diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 99067bdb1..39457a1a7 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -5,6 +5,7 @@ on: branches: - 'main' - 'develop-[0-9]+.[0-9]+.[0-9]+' + - 'build-doc-[0-9]+.[0-9]+.[0-9]+-[a-zA-Z]+' schedule: - cron: '0 8 * * *' @@ -41,6 +42,7 @@ jobs: VERSION='${{ github.ref_name }}' [ "$VERSION" == main ] && { VERSION=latest; ALIAS='main master'; } VERSION="${VERSION#develop-}" + VERSION="${VERSION#build-doc-}" mike deploy --push --update-aliases "$VERSION" $ALIAS mike set-default --push latest diff --git a/.gitignore b/.gitignore index 2ad11752a..44e3d77a5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,15 +13,15 @@ venv # excluded paths /data/ +/model/ /logs/ /jobs/ /audit/ +/localfs/ .vscode/* /temp/ /tmp /worker/ -/provider_registrar/ -/model_local_cache/ *.db *.db-journal *.whl @@ -31,3 +31,14 @@ venv # doc /site/ + +/python/fate_flow/data +/python/fate_flow/model +/python/fate_flow/logs +/python/fate_flow/jobs +/python/fate_flow/localfs +/python/fate_flow/*.env +/python/fate_flow/conf +/python/build +/python/dist +/python/*.egg-info \ No newline at end of file diff --git a/README.md b/README.md index 95cf06702..25ea2995a 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,6 @@ Providing production-level service capabilities: - High Availability - CLI, REST API, Python API -For detailed introduction, please refer to [FATE Flow Overall Design](https://federatedai.github.io/FATE-Flow/latest/fate_flow/#overall-design) - ## Deployment Please refer to [FATE](https://github.com/FederatedAI/FATE) diff --git a/README.zh.md b/README.zh.md index b1aaaece9..13da9e4cb 100644 --- a/README.zh.md +++ b/README.zh.md @@ -4,7 +4,7 @@ FATE Flow是一个联邦学习端到端全流程的多方联合任务安全调度平台, 基于: -- [共享状态调度架构](https://storage.googleapis.com/pub-tools-public-publication-data/pdf/41684.pdf) +- 共享状态调度架构 - 跨数据中心的多方安全通信 提供生产级服务能力: @@ -20,7 +20,6 @@ FATE Flow是一个联邦学习端到端全流程的多方联合任务安全调 - 系统高可用 - CLI、REST API、Python API -详细介绍请参考[FATE Flow整体设计](https://federatedai.github.io/FATE-Flow/latest/zh/fate_flow/) ## 部署 diff --git a/RELEASE.md b/RELEASE.md index 06ec68461..67058bb6b 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,259 +1,15 @@ -# Release 1.11.2 -## Major Features and Improvements -* Support real-time log retrieval and display for FATE LLM tasks. -* Optimize the logic of the job clean interface. - -## Bug Fixes -* Fix the thread accumulation caused by the session cleanup timeout bug. - -# Release 1.11.1 -## Major Features and Improvements -* Support distributed training with multiple gpus for FATE-LLM by Eggroll - -## Bug Fixes -* Fix hadoop connection failures in some scenarios -* Fix spark config in role does not take effect - -# Release 1.11.0 -## Major Features and Improvements -* Add data table preview query interface - -## Bug Fixes -* Fix the performance problems of upload and reader in processing large amounts of data -* Fix online inference cannot be done after model migration bug -* Fix the model cannot be saved to the specified database bug -* Fix reader data preview display bug - - -# Release 1.10.1 -## Major Features and Improvements -* Optimize table info API - - -# Release 1.10.0 -## Major Features and Improvements -* Add connection test API -* May configure gRPC message size limit -## Bug Fixes -* Fix module duplication issue in model - -# Release 1.9.1 -## Bug Fixes -* Fix parameter inheritance when loading non-model modules from ModelLoader -* Fix job inheritance after adding or removing roles from training configuration -* Fix delimiter error in uploaded/downloaded data -* Fix anonymous feature name renewal - -# Release 1.9.0 -## Major Features and Improvements -* Support high availability and load balancing to improve system availability and stability -* Added support for site authentication and data set authority authentication, and supports hook mode for users to customize authentication schemes -* Component registration optimization, support participants to use different versions of algorithm components -* Upload, reader support feature anonymity, support specifying id column -* Scheduling optimization, asynchronous time-consuming operations, component scheduling performance improved by more than 5 times This optimization obvious benefits for multi-component tasks -* Added component ApiReader to get feature data by id -* Model storage optimization, support model data synchronization between local and other storage -* The scheduler now can obtain the error information from other participant's algorithm components - -# Release 1.8.0 -## Major Features and Improvements -* Optimize the model migration function to reduce user operation steps; -* Add version compatibility check in component center to support multiple parties to use different versions; -* Add data table disable/enable function, and support batch delete disable table - -# Release 1.7.2 -## Major Features and Improvements -* Separate the base connection address of the data storage table from the data table information, and compatible with historical versions; -* Optimize the component output data download interface. - -# Release 1.7.1 -## Major Features and Improvements -* Added the writer component, which supports exporting data to mysql and saving data as a new table; -* Added job reuse function, which supports the reuse of successful status components of historical tasks in new jobs; -* Optimize the time-consuming problem of submitting tasks and the time-consuming problem of stopping tasks; -* Component registration supports automatic setting of PYTHONPYTH. - -## Bug Fixes -* Fix the problem of OOM when uploading hdfs table; -* Fix the problem of incompatibility with the old version of serving; -* The parameter partitions of the toy test is set to 4, and a timeout prompt is added. - -# Release 1.7.0 - -## Major Features and Improvements - -* Independent repository instead of all code in the main FATE repository -* Component registry, which can hot load many different versions of component packages at the same time -* Hot update of component parameters, component-specific reruns, automatic reruns -* Model Checkpoint to support task hot start, model deployment and other -* Data, Model and Cache can be reused between jobs -* Reader component supports more data sources, such as MySQL, Hive -* Realtime recording of dataset usage derivation routes -* Multi-party permission control for datasets -* Automatic push to reliable storage when model deployment, support Tencent Cloud COS, MySQL, Redis -* REST API authentication - -## Bug Fixes - -# Release 1.6.1 -## Major Features and Improvements -* Support mysql storage engine; -* Added service registry interface; -* Added service query interface; -* Support fate on WeDataSphere mode -* Add lock when writing `model_local_cache` -* Register the model download urls to zookeeper - -## Bug Fixes -* Fix job id length no more than 25 limitation - - -# Release 1.5.2 -## Major Features and Improvements -* Read data from mysql with ‘table bind’ command to map source table to FATE table -* FATE cluster push model for one-to-multiple FATE Serving clusters in one party - -## Bug Fixes -* Fix job id length no more than 25 limitation - - -# Release 1.5.1 -## Major Features and Improvements -* Optimize the model center, reconstruct publishing model, support deploy, load, bind, migrate operations, and add new interfaces such as model info -* Improve identity authentication and resource authorization, support party identity verification, and participate in the authorization of roles and components -* Optimize and fix resource manager, add task_cores job parameters to adapt to different computing engines - -## Deploy -* Support 1.5.0 retain data upgrade to 1.5.1 - -## Bug Fixes -* Fix job clean CLI - - -# Release 1.5.0（LTS） -## Major Features and Improvements -* Brand new scheduling framework based on global state and optimistic concurrency control and support multiple scheduler -* Upgraded task scheduling: multi-model output for component, executing component in parallel, component rerun -* Add new DSL v2 which significantly improves user experiences in comparison to DSL v1. Several syntax error detection functions are supported in v2. Now DSL v1 and v2 are - compatible in the current FATE version -* Enhanced resource scheduling: remove limit on job number, base on cores, memory and working node according to different computing engine supports -* Add model registry, supports model query, import/export, model transfer between clusters -* Add Reader component: automatically dump input data to FATE-compatible format and cluster storage engine; now data from HDFS -* Refactor submit job configuration's parameters setting, support different parties use different job parameters when using dsl V2. - -## Client -* Brand new CLI v2 with easy independent installation, user-friendly programming syntax & command-line prompt -* Support FLOW python language SDK - - -# Release 1.4.4 -## Major Features and Improvements -* Task Executor supports monkey patch -* Add forward API - - -# Release 1.4.2 -## Major Features and Improvements -* Distinguish between user stop job and system stop job; -* Optimized some logs; -* Optimize zookeeper configuration -* The model supports persistent storage to mysql -* Push the model to the online service to support the specified storage address (local file and FATEFlowServer interface) - - -# Release 1.4.1 -## Major Features and Improvements -* Allow the host to stop the job -* Optimize the task queue -* Automatically align the input table partitions of all participants when the job is running -* Fate flow client large file upload optimization -* Fixed some bugs with abnormal status - - -# Release 1.4.0 -## Major Features and Improvements -* Refactoring model management, native file directory storage, storage structure is more flexible, more information -* Support model import and export, store and restore with reliable distributed system(Redis is currently supported) -* Using MySQL instead of Redis to implement Job Queue, reducing system complexity -* Support for uploading client local files -* Automatically detects the existence of the table and provides the destroy option -* Separate system, algorithm, scheduling command log, scheduling command log can be independently audited - - -# Release 1.3.1 -## Major Features and Improvements -## Deploy -* Support deploying by MacOS -* Support using external db -* Deploy JDK and Python environments on demand -* Improve MySQL and FATE Flow service.sh -* Support more custom deployment configurations in the default_configurations.sh, such as ssh_port, mysql_port and so one. - -# Release 1.3.0 -## Major Features and Improvements -* Add clean job CLI for cleaning output and intermediate results, including data, metrics and sessions -* Support for obtaining table namespace and name of output data via CLI -* Fix KillJob unsuccessful execution in some special cases -* Improve log system, add more exception and run time status prompts - - -# Release 1.2.0 -## Major Features and Improvements -* Add data management module for recording the uploaded data tables and the outputs of the model in the job running, and for querying and cleaning up CLI. -* Support registration center for simplifying communication configuration between FATEFlow and FATEServing -* Restruct model release logic, FATE_Flow pushes model directly to FATE-Serving. Decouple FATE-Serving and Eggroll, and the offline and online architectures are connected only by FATE-Flow. -* Provide CLI to query data upload record -* Upload and download data support progress statistics by line -* Add some abnormal diagnosis tips -* Support adding note information to job - -## Deploy -* Fix bugs in EggRoll startup script, add mysql, redis startup options. -* Disable host name resolution configuration for mysql service. -* The version number of each module of the software packaging script is updated using the automatic acquisition mode. - - -# Release 1.1.1 -## Major Features and Improvements -* Add cluster deployment support based on ubuntu operating system。 -* Support intermediate data cleanup after the task ends -* Optimizing the deployment process - - -## Bug Fixes -* Fix a bug in download api -* Fix bugs of spark-backend - - -# Release 1.1 -## Major Features and Improvements -* Upload and Download support CLI for querying job status -* Support for canceling waiting job -* Support for setting job timeout -* Support for storing a job scheduling log in the job log folder -* Add authentication control Beta version, including component, command, role - - -# Release 1.0.2 -## Major Features and Improvements -* Python and JDK environment are required only for running standalone version quick experiment -* Support cluster version docker deployment -* Add deployment guide in Chinese -* Standalone version job for quick experiment is supported when cluster version deployed. -* Python service log will remain for 14 days now. - - -# Release 1.0.1 -## Bug Fixes -* Support upload file in version argument -* Support get serviceRoleName from configuration - - -# Release 1.0 -## Major Features and Improvements -* DAG defines Pipeline -* Federated Multi-party asymmetric DSL parser -* Federated Learning lifecycle management -* Federated Task collaborative scheduling -* Tracking for data, metric, model and so on -* Federated Multi-party model management \ No newline at end of file +## Release 2.0.0 +### Major Features and Improvements +* Adapted to new scalable and standardized federated DSL IR +* Built an interconnected scheduling layer framework, supported the BFIA protocol +* Optimized process scheduling, with scheduling separated and customizable, and added priority scheduling +* Optimized algorithm component scheduling，support container-level algorithm loading， enhancing support for cross-platform heterogeneous scenarios +* Optimized multi-version algorithm component registration, supporting registration for mode of components +* Federated DSL IR extension enhancement: supports multi-party asymmetric scheduling +* Optimized client authentication logic, supporting permission management for multiple clients +* Optimized RESTful interface, making parameter fields and types, return fields, and status codes clearer +* Added OFX(Open Flow Exchange) module: encapsulated scheduling client to allow cross-platform scheduling +* Supported the new communication engine OSX, while remaining compatible with all engines from FATE Flow 1.x +* Decoupled the System Layer and the Algorithm Layer, with system configuration moved from the FATE repository to the Flow repository +* Published FATE Flow package to PyPI and added service-level CLI for service management +* Migrated major functionality from FATE Flow 1.x diff --git a/bin/init_env.sh b/bin/init_env.sh new file mode 100644 index 000000000..5213ab258 --- /dev/null +++ b/bin/init_env.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +fate_project_base=$(cd `dirname "$(realpath "${BASH_SOURCE[0]:-${(%):-%x}}")"`; cd ../;cd ../;pwd) +export FATE_PROJECT_BASE=$fate_project_base +export PYTHONPATH= +export SPARK_HOME= +venv= + +source ${venv}/bin/activate + diff --git a/bin/service.sh b/bin/service.sh index 6b183e125..083d7bfab 100644 --- a/bin/service.sh +++ b/bin/service.sh @@ -1,5 +1,4 @@ #!/bin/bash - # # Copyright 2019 The FATE Authors. All Rights Reserved. # @@ -15,171 +14,392 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# ----------------------------------------------------------------------------- +# Service Control Script for a FATE Flow Server Application +# ----------------------------------------------------------------------------- +# +# This script is used to manage the lifecycle (start, stop, restart, and check status) of a server application. +# The server application listens on two ports: an HTTP port and a gRPC port. +# The settings for these ports, as well as other configurations, are read from a YAML file. +# +# Dependencies: +# - lsof: To check which processes are listening on which ports. +# - sed, awk: For text processing, mainly for parsing the YAML configuration. +# +# Usage: +# ./service.sh {start|stop|status|restart [sleep_time]} +# sleep_time: Optional. Number of seconds to wait between stop and start during restart. Default is 10 seconds. +# +# Assumptions: +# - The script assumes the presence of a configuration file named 'service_conf.yaml' in a relative directory. +# - The configuration file is structured in a specific way that the parsing logic expects. +# +# ----------------------------------------------------------------------------- + +# --------------- Color Definitions --------------- +esc_c=$(tput sgr0) +error_c=$(tput setaf 1) +ok_c=$(tput setaf 2) +highlight_c=$(tput setaf 3) + +# --------------- Logging Functions --------------- +print_info() { + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + local overwrite=$2 -if [[ -z "${FATE_PROJECT_BASE}" ]]; then - PROJECT_BASE=$(cd "$(dirname "$0")";cd ../;cd ../;pwd) -else - PROJECT_BASE="${FATE_PROJECT_BASE}" -fi -FATE_FLOW_BASE=${PROJECT_BASE}/fateflow -echo "PROJECT_BASE: "${PROJECT_BASE} - -# source init_env.sh -INI_ENV_SCRIPT=${PROJECT_BASE}/bin/init_env.sh -if test -f "${INI_ENV_SCRIPT}"; then - source ${PROJECT_BASE}/bin/init_env.sh - echo "PYTHONPATH: "${PYTHONPATH} -else - echo "file not found: ${INI_ENV_SCRIPT}" - exit -fi - -log_dir=${FATE_FLOW_BASE}/logs - -module=fate_flow_server.py - -parse_yaml() { - local prefix=$2 - local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034') - sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ - -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 | - awk -F$fs '{ - indent = length($1)/2; - vname[indent] = $2; - for (i in vname) {if (i > indent) {delete vname[i]}} - if (length($3) > 0) { - vn=""; for (i=0; i/dev/null; then + print_error "Missing dependency" "$dep" + missing_deps=1 + fi + done -mklogsdir() { - if [[ ! -d $log_dir ]]; then - mkdir -p $log_dir + if [ "$missing_deps" -eq 1 ]; then + print_error "Please install the missing dependencies and try again." + exit 1 fi } -status() { - getpid - if [[ -n ${pid} ]]; then - echo "status:`ps aux | grep ${pid} | grep -v grep`" - lsof -i:${service_config_fateflow_http_port} | grep 'LISTEN' - lsof -i:${service_config_fateflow_grpc_port} | grep 'LISTEN' +# Get the PID of the process using a specific port +get_pid() { + local port=$1 + lsof -i:${port} | grep 'LISTEN' | awk 'NR==1 {print $2}' +} + +# Extract the specified port from a specified section of the YAML file +get_port_from_yaml() { + local yaml_file=$1 + local section=$2 + local port_key=$3 + local s='[[:space:]]*' + local w='[a-zA-Z0-9_]*' + local fs=$(echo @ | tr @ '\034') # Set fs back to the ASCII "file separator" + + sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ + -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $yaml_file | + awk -F$fs -v section="$section" -v port_key="$port_key" ' + { + indent = length($1)/2; + vname[indent] = $2; + for (i in vname) {if (i > indent) {delete vname[i]}} + if (length($3) > 0) { + vn=""; + for (i=0; i> "${log_dir}/console.log" 2>>"${log_dir}/error.log" - unset FATE_PROJECT_BASE - else - export FATE_PROJECT_BASE=${PROJECT_BASE} - nohup python ${FATE_FLOW_BASE}/python/fate_flow/fate_flow_server.py >> "${log_dir}/console.log" 2>>"${log_dir}/error.log" & - unset FATE_PROJECT_BASE +# --------------- Functions for start--------------- +# Check if the service is up and running +check_service_up() { + local pid=$1 + local timeout_ms=$2 + local interval_ms=$3 + local http_port=$4 + local grpc_port=$5 + local elapsed_ms=0 + local spin_chars="/-\\|" + + while ((elapsed_ms < timeout_ms)); do + if ! kill -0 "${pid}" 2>/dev/null; then + echo "Process with PID ${pid} is not running." + echo + return 1 fi - for((i=1;i<=100;i++)); - do - sleep 0.1 - getpid - if [[ -n ${pid} ]]; then - echo "service start sucessfully. pid: ${pid}" - return - fi - done - if [[ -n ${pid} ]]; then - echo "service start sucessfully. pid: ${pid}" - else - echo "service start failed, please check ${log_dir}/error.log and ${log_dir}/console.log" + + if lsof -i :${http_port} | grep -q LISTEN && lsof -i :${grpc_port} | grep -q LISTEN; then + echo "Service started successfully!" + echo + return 0 fi + + local char_index=$((elapsed_ms / interval_ms % ${#spin_chars})) + local char="${spin_chars:char_index:1}" + printf "[%s][MS] %s\r" "$(date '+%Y-%m-%d %H:%M:%S')" "$char" + sleep $((interval_ms / 1000)).$((interval_ms % 1000)) + elapsed_ms=$((elapsed_ms + interval_ms)) + done + echo "Service did not start up within the expected time." + echo + return 1 +} + +# Draw a progress bar for visual feedback +draw_progress_bar() { + local completed=$1 + local total=$2 + local msg="$3" + local progress_bar="[" + + # Print completed part + for ((i = 0; i < completed; i++)); do + progress_bar+=" " + done + + # Print pending part + for ((i = completed; i < total; i++)); do + progress_bar+="-" + done + progress_bar+="]${msg}" + print_info "$progress_bar" "overwrite" +} + +# Checks if a port is active and returns the PID of the process using it. +# Parameters: +# $1 - The port number to check. +# Returns: +# PID of the process using the port, or an empty string if the port is not active. +check_port_active() { + local port=$1 + lsof -i:${port} | grep 'LISTEN' | awk 'NR==1 {print $2}' +} + +# Start service +start() { + print_info "--------------------------------starting--------------------------------" + print_info "Verifying if HTTP port ${highlight_c}${http_port}${esc_c} is not active..." + pid1=$(check_port_active $http_port) + if [ -n "${pid1}" ]; then + print_error "HTTP port ${highlight_c}${http_port}${esc_c} is already active. Process ID (PID): ${highlight_c}${pid1}${esc_c}" + exit 1 + else + print_ok "HTTP port ${highlight_c}${http_port}${esc_c} not active" + fi + + print_info "Verifying if gRPC port ${highlight_c}${grpc_port}${esc_c} is not active..." + pid2=$(check_port_active $grpc_port) + if [ -n "${pid2}" ]; then + print_error "gRPC port ${highlight_c}${grpc_port}${esc_c} is already active. Process ID (PID): ${highlight_c}${pid2}${esc_c}" + exit 1 + else + print_ok "gRPC port ${highlight_c}${grpc_port}${esc_c} not active" + fi + + print_info "Starting services..." + local startup_error_tmp=$(mktemp) + if [ "$1" = "front" ]; then + exec FATE_PROJECT_BASE="${PROJECT_BASE}" python "${FATE_FLOW_BASE}/python/fate_flow/fate_flow_server.py" >>"${LOG_STDOUT}" 2>>"${LOG_STDERR}" else - echo "service already started. pid: ${pid}" + export FATE_PROJECT_BASE="${PROJECT_BASE}" + nohup python "${FATE_FLOW_BASE}/python/fate_flow/fate_flow_server.py" >>"${LOG_STDOUT}" 2>>"${LOG_STDERR}" & + unset FATE_PROJECT_BASE + pid=$! + print_info "Process ID (PID): ${highlight_c}${pid}${esc_c}" + if ! check_service_up "${pid}" 5000 250 ${http_port} ${grpc_port}; then + print_info "stderr:" + cat "${startup_error_tmp}" + rm "${startup_error_tmp}" + print_info "Please check ${LOG_STDERR} and ${LOG_STDOUT} for more details" + exit 1 + fi fi } -stop() { - getpid - if [[ -n ${pid} ]]; then - echo "killing: `ps aux | grep ${pid} | grep -v grep`" - for((i=1;i<=100;i++)); - do +# --------------- Functions for stop--------------- +# Function to kill a process +kill_process() { + local pid=$1 + local signal=$2 + kill ${signal} "${pid}" 2>/dev/null +} + +# Stop service +stop_port() { + local port=$1 + local name=$2 + local pid=$(get_pid ${port}) + + print_info "Stopping $name ${highlight_c}${port}${esc_c}..." + if [ -n "${pid}" ]; then + for _ in {1..100}; do sleep 0.1 - kill ${pid} - getpid - if [[ ! -n ${pid} ]]; then - echo "killed by SIGTERM" + kill_process "${pid}" + pid=$(get_pid ${port}) + if [ -z "${pid}" ]; then + print_ok "Stop $name ${highlight_c}${port}${esc_c} success (SIGTERM)" return fi done - kill -9 ${pid} - if [[ $? -eq 0 ]]; then - echo "killed by SIGKILL" + kill_process "${pid}" -9 && print_ok "Stop port success (SIGKILL)" || print_error "Stop port failed" + else + print_ok "Stop $name ${highlight_c}${port}${esc_c} success(NOT ACTIVE)" + fi +} +stop() { + print_info "--------------------------------stopping--------------------------------" + stop_port ${http_port} "HTTP port" + stop_port ${grpc_port} "gRPC port" +} + +# --------------- Functions for status--------------- +# Check the status of the service +status() { + print_info "---------------------------------status---------------------------------" + # Check http_port + pid1=$(check_port_active $http_port) + if [ -n "${pid1}" ]; then + print_ok "Check http port ${highlight_c}${http_port}${esc_c} is active: PID=${highlight_c}${pid1}${esc_c}" + else + print_error "http port not active" + fi + + # Check grpc_port + pid2=$(check_port_active $grpc_port) + if [ -n "${pid2}" ]; then + print_ok "Check grpc port ${highlight_c}${grpc_port}${esc_c} is active: PID=${highlight_c}${pid2}${esc_c}" + else + print_error "grpc port not active" + fi + + # Check if both PIDs are the same + if [ -n "${pid1}" ] && [ -n "${pid2}" ]; then + if [ "${pid1}" == "${pid2}" ]; then + print_ok "Check http port and grpc port from same process: PID=${highlight_c}${pid2}${esc_c}" else - echo "kill error" + print_error "Found http port and grpc port active but from different process: ${highlight_c}${pid2}${esc_c}!=${highlight_c}${pid2}${esc_c}" fi - else - echo "service not running" fi } +# --------------- Functions for info--------------- +# Print usage information for the script +print_usage() { + echo -e "${ok_c}FATE Flow${esc_c}" + echo "---------" + echo -e "${ok_c}Usage:${esc_c}" + echo -e " $0 start - Start the server application." + echo -e " $0 stop - Stop the server application." + echo -e " $0 status - Check and report the status of the server application." + echo -e " $0 restart [time] - Restart the server application. Optionally, specify a sleep time (in seconds) between stop and start." + echo "" + echo -e "${ok_c}Examples:${esc_c}" + echo " $0 start" + echo " $0 restart 5" + echo "" + echo -e "${ok_c}Notes:${esc_c}" + echo " - The restart command, if given an optional sleep time, will wait for the specified number of seconds between stopping and starting the service." + echo " If not provided, it defaults to 10 seconds." + echo " - Ensure that the required configuration file 'service_conf.yaml' is properly set up in the expected directory." + echo "" + echo "For more detailed information, refer to the script's documentation or visit the official documentation website." +} +# --------------- Main--------------- +# Main case for control case "$1" in start) + load_config start status ;; - starting) + load_config start front ;; - stop) + load_config stop ;; - status) + load_config status ;; - restart) + load_config stop - sleep 10 + sleep_time=${2:-5} + print_info "Waiting ${sleep_time} seconds" + sleep $sleep_time start status ;; *) - echo "usage: $0 {start|stop|status|restart}" - exit -1 + print_usage + exit 1 + ;; esac diff --git a/conf/casbin_model.conf b/conf/casbin_model.conf index 60a4e14f9..71159e387 100644 --- a/conf/casbin_model.conf +++ b/conf/casbin_model.conf @@ -1,11 +1,14 @@ [request_definition] -r = party_id, type, value +r = sub, obj, act [policy_definition] -p = party_id, type, value +p = sub, obj, act + +[role_definition] +g = _, _ [policy_effect] e = some(where (p.eft == allow)) [matchers] -m = r.party_id == p.party_id && r.type == p.type && r.value == p.value \ No newline at end of file +m = g(r.sub, p.sub) && r.obj == p.obj && r.act == p.act \ No newline at end of file diff --git a/conf/component_registry.json b/conf/component_registry.json deleted file mode 100644 index 6061b6494..000000000 --- a/conf/component_registry.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "components": { - }, - "providers": { - }, - "default_settings": { - "fate_flow":{ - "default_version_key": "FATEFlow" - }, - "fate": { - "default_version_key": "FATE" - }, - "class_path": { - "interface": "components.components.Components", - "feature_instance": "feature.instance.Instance", - "feature_vector": "feature.sparse_vector.SparseVector", - "model": "protobuf.generated", - "model_migrate": "protobuf.model_migrate.model_migrate", - "homo_model_convert": "protobuf.homo_model_convert.homo_model_convert", - "anonymous_generator": "util.anonymous_generator_util.Anonymous", - "data_format": "util.data_format_preprocess.DataFormatPreProcess", - "hetero_model_merge": "protobuf.model_merge.merge_hetero_models.hetero_model_merge", - "extract_woe_array_dict": "protobuf.model_migrate.binning_model_migrate.extract_woe_array_dict", - "merge_woe_array_dict": "protobuf.model_migrate.binning_model_migrate.merge_woe_array_dict" - } - } -} diff --git a/conf/incompatible_version.yaml b/conf/incompatible_version.yaml deleted file mode 100644 index 448a49bd5..000000000 --- a/conf/incompatible_version.yaml +++ /dev/null @@ -1,5 +0,0 @@ -FATE: - 1.7: <1.7.0, >=1.8.0 - 1.7.2: <=1.7.0, 1.7.1, 1.7.1.1, >=1.8.0 - 1.8: <1.8.0, >=1.9.0 - 1.9: <1.9.0 \ No newline at end of file diff --git a/conf/job_default_config.yaml b/conf/job_default_config.yaml index c557893db..c72e35bd7 100644 --- a/conf/job_default_config.yaml +++ b/conf/job_default_config.yaml @@ -1,35 +1,18 @@ -# component provider, relative path to get_fate_python_directory -default_component_provider_path: federatedml - # resource -total_cores_overweight_percent: 1 # 1 means no overweight -total_memory_overweight_percent: 1 # 1 means no overweight -task_parallelism: 1 -task_cores: 4 -task_memory: 0 # mb -max_cores_percent_per_job: 1 # 1 means total - -# scheduling -job_timeout: 259200 # s +job_cores: 4 +computing_partitions: 8 +task_run: + spark: + num-executors: 2 + executor-cores: 2 + eggroll: + cores: 4 + standalone: + cores: 4 +task_timeout: 259200 # s remote_request_timeout: 30000 # ms federated_command_trys: 3 -end_status_job_scheduling_time_limit: 300000 # ms -end_status_job_scheduling_updates: 1 auto_retries: 0 -auto_retry_delay: 1 #seconds -# It can also be specified in the job configuration using the federated_status_collect_type parameter -federated_status_collect_type: PUSH -detect_connect_max_retry_count: 3 -detect_connect_long_retry_count: 2 - -task_process_classpath: true - -# upload -upload_block_max_bytes: 104857600 # bytes - -#component output -output_data_summary_count_limit: 100 - -# gpu -task_world_size: 2 -resource_waiting_timeout: 21600 # s \ No newline at end of file +sync_type: callback # poll or callback +task_device: + type: CPU diff --git a/conf/permission_casbin_model.conf b/conf/permission_casbin_model.conf new file mode 100644 index 000000000..60a4e14f9 --- /dev/null +++ b/conf/permission_casbin_model.conf @@ -0,0 +1,11 @@ +[request_definition] +r = party_id, type, value + +[policy_definition] +p = party_id, type, value + +[policy_effect] +e = some(where (p.eft == allow)) + +[matchers] +m = r.party_id == p.party_id && r.type == p.type && r.value == p.value \ No newline at end of file diff --git a/conf/pulsar_route_table.yaml b/conf/pulsar_route_table.yaml new file mode 100644 index 000000000..730b5001a --- /dev/null +++ b/conf/pulsar_route_table.yaml @@ -0,0 +1,22 @@ +9999: + # host can be a domain like 9999.fate.org + host: 127.0.0.1 + port: 6650 + sslPort: 6651 + # set proxy address for this pulsar cluster + proxy: "" + +10000: + # host can be a domain like 10000.fate.org + host: 127.0.0.1 + port: 6650 + sslPort: 6651 + proxy: "" + +default: + # compose host and proxy for party that does not exist in route table + # in this example, the host for party 8888 will be 8888.fate.org + proxy: "proxy.fate.org:443" + domain: "fate.org" + brokerPort: 6650 + brokerSslPort: 6651 \ No newline at end of file diff --git a/conf/rabbitmq_route_table.yaml b/conf/rabbitmq_route_table.yaml new file mode 100644 index 000000000..b1fee9d09 --- /dev/null +++ b/conf/rabbitmq_route_table.yaml @@ -0,0 +1,6 @@ +9999: + host: 192.168.0.4 + port: 5672 +10000: + host: 192.168.0.3 + port: 5672 diff --git a/conf/service_conf.yaml b/conf/service_conf.yaml new file mode 100644 index 000000000..4501f3335 --- /dev/null +++ b/conf/service_conf.yaml @@ -0,0 +1,120 @@ +party_id: "9999" +use_registry: false +# DEBUG 10/INFO 20 +log_level: 20 +encrypt: + key_0: + module: fate_flow.hub.encrypt.password_encrypt#pwdecrypt + # base on: fate_flow/conf/ + private_path: private_key.pem +fateflow: + host: 127.0.0.1 + http_port: 9380 + grpc_port: 9360 + proxy_name: osx +# nginx: +# host: +# http_port: +# grpc_port: +database: + engine: sqlite + # encrypt passwd key + decrypt_key: + mysql: + name: fate_flow + user: fate + passwd: fate + host: 127.0.0.1 + port: 3306 + max_connections: 100 + stale_timeout: 30 + sqlite: + # default fate_flow/runtime/system_settings: SQLITE_PATH + # /xxx/xxx.sqlite + path: +default_engines: + computing: standalone + federation: standalone + storage: standalone +default_provider: + name: fate + # version default: fateflow.env + version: + device: local +computing: + standalone: + cores: 32 + eggroll: + cores: 32 + nodes: 1 + # cluster manager host and port + host: 127.0.0.1 + port: 4670 + spark: + # default use SPARK_HOME environment variable + home: + cores: 32 +federation: + osx: + host: 127.0.0.1 + port: 9370 + # stream or queue + mode: stream +# pulsar: +# host: 192.168.0.5 +# port: 6650 +# mng_port: 8080 +# cluster: standalone +# tenant: fl-tenant +# topic_ttl: 30 +# # default conf/pulsar_route_table.yaml +# route_table: +# # mode: replication / client, default: replication +# mode: replication +# rabbitmq: +# host: 192.168.0.4 +# mng_port: 12345 +# port: 5672 +# user: fate +# password: fate +# # default conf/rabbitmq_route_table.yaml +# route_table: +# # mode: replication / client, default: replication +# mode: replication +storage: + hdfs: + name_node: hdfs://fate-cluster +hook_module: + client_authentication: fate_flow.hook.flow.client_authentication + site_authentication: fate_flow.hook.flow.site_authentication + permission: fate_flow.hook.flow.permission +authentication: + client: false + site: false + permission: false +model_store: + engine: file + # encrypt passwd key + decrypt_key: + file: + # default fate_flow/runtime/system_settings: MODEL_STORE_PATH + path: + mysql: + name: fate_flow + user: fate + passwd: fate + host: 127.0.0.1 + port: 3306 + max_connections: 100 + stale_timeout: 30 + tencent_cos: + Region: + SecretId: + SecretKey: + Bucket: +zookeeper: + hosts: + - 127.0.0.1:2181 + use_acl: true + user: fate + password: fate diff --git a/conf/template_info.yaml b/conf/template_info.yaml deleted file mode 100644 index ae5140e78..000000000 --- a/conf/template_info.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# base dir: fateflow -template_path: - fate_examples: ../examples - fateflow_examples: examples -template_data: - base_dir: ../examples/data - min_data: ['breast_hetero_guest.csv', 'breast_hetero_host.csv', 'default_credit_hetero_guest.csv', 'default_credit_hetero_host.csv'] - -delete_path: - fateflow_examples: [data] \ No newline at end of file diff --git a/doc/bfia_access.zh.md b/doc/bfia_access.zh.md new file mode 100644 index 000000000..dd93ad64a --- /dev/null +++ b/doc/bfia_access.zh.md @@ -0,0 +1,9 @@ +# 整体设计 + +## 1. 逻辑架构 + +- Pipeline构造应用层互联互通 +- DSL IR定义统一作业配置标准 +- FATE Flow调度抽象API +- 算法容器化调度 +- 多适配器模式 diff --git a/doc/build/build.py b/doc/build/build.py new file mode 100644 index 000000000..39eac2b4e --- /dev/null +++ b/doc/build/build.py @@ -0,0 +1,38 @@ +import json +import os.path +import subprocess +import sys +import threading + +import requests + + +def run_script(script_path, *args): + result = subprocess.run(['python', script_path, *args]) + return result.stderr + + +if __name__ == '__main__': + base_dir = os.path.dirname(__file__) + build_path = os.path.join(base_dir, 'build_swagger_server.py') + + thread = threading.Thread(target=run_script, args=(build_path,)) + thread.start() + # + thread.join() + build_path = os.path.join(base_dir, 'swagger_server.py') + port = "50000" + server = threading.Thread(target=run_script, args=(build_path, port)) + + result = server.start() + + import time + time.sleep(3) + data = requests.get(url=f"http://127.0.0.1:{port}/swagger.json").text + data = json.loads(data) + swagger_file = os.path.join(os.path.dirname(base_dir), "swagger", "swagger.json") + os.makedirs(os.path.dirname(swagger_file), exist_ok=True) + with open(swagger_file, "w") as fw: + json.dump(data, fw, indent=4) + print("build success!") + sys.exit() diff --git a/doc/build/build_swagger_server.py b/doc/build/build_swagger_server.py new file mode 100644 index 000000000..5c3990e15 --- /dev/null +++ b/doc/build/build_swagger_server.py @@ -0,0 +1,166 @@ +import ast +import os.path +import re +from importlib.util import spec_from_file_location, module_from_spec +from pathlib import Path + +import fate_flow +from fate_flow.runtime.system_settings import HOST, HTTP_PORT, API_VERSION + +base_path = f"/{API_VERSION}" +FATE_FLOW_HOME = os.path.dirname(fate_flow.__file__) +DOC_BASE = os.path.join(os.path.dirname(os.path.dirname(FATE_FLOW_HOME)), "doc", "build") +swagger_py_file = os.path.join(DOC_BASE, "swagger_server.py") + + +def search_pages_path(pages_dir): + return [path for path in pages_dir.glob('*_app.py') if not path.name.startswith('.')] + + +def read_desc_script(files): + with open(files, "r") as file: + content = file.read() + + pattern = r'(\w+)\s*=\s*"([^"]+)"' + variables = dict(re.findall(pattern, content)) + return variables + + +def scan_client_app(file_path, variables): + function_info = {} + for _path in file_path: + page_name = _path.stem.rstrip('app').rstrip("_") + module_name = '.'.join(_path.parts[_path.parts.index('apps') - 1:-1] + (page_name,)) + spec = spec_from_file_location(module_name, _path) + page = module_from_spec(spec) + page_name = getattr(page, 'page_name', page_name) + if page_name not in function_info: + function_info[page_name] = [] + with open(str(_path), 'r') as file: + tree = ast.parse(file.read()) + + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef): + function_name = node.name + function_params = [] + function_route = None + function_method = None + function_params_desc = {} + + for arg in node.args.args: + function_params.append(arg.arg) + + for decorator in node.decorator_list: + if isinstance(decorator, ast.Call) and isinstance(decorator.func, ast.Attribute): + if decorator.func.attr == 'route': + function_route = decorator.args[0].s + if isinstance(decorator.keywords, list): + for keyword in decorator.keywords: + if keyword.arg == 'methods': + function_method = keyword.value.elts[0].s + + else: + params_value = "" + params_name = "" + for key in decorator.keywords: + if key.arg == 'desc': + params_value = key.value.id + else: + params_name = key.arg + + if params_name: + function_params_desc[params_name] = variables.get(params_value, "") + function_info[page_name].append({ + 'function_name': function_name, + 'function_route': function_route, + 'function_method': function_method, + 'function_params_desc': function_params_desc, + }) + + return function_info + + +def generate_transfer_doc(function_info): + script = f""" +from flask import Flask +from flask_restx import Api, Resource, Swagger +from werkzeug.utils import cached_property + + +class RSwagger(Swagger): + def as_dict_v2(self): + _dict = self.as_dict() + _dict["basePath"] = "{base_path}" + return _dict + + def operation_id_for(self, doc, method): + return ( + doc[method].get("operationId") + if "operationId" in doc[method] + else self.api.default_id(doc["name"], method) + ) + + def description_for(self, doc, method): + return doc[method].get("description") + + +class RApi(Api): + @cached_property + def __schema__(self): + if not self._schema: + try: + self._schema = RSwagger(self).as_dict_v2() + except Exception: + msg = "Unable to render schema" + log.exception(msg) + return msg + return self._schema + + +app = Flask(__name__) +api = RApi(app, version="{fate_flow.__version__}", title="FATE Flow restful api") +""" + + for page_name in function_info.keys(): + script += f""" +{page_name} = api.namespace("{page_name}", description="{page_name}-Related Operations") +""" + for page_name, infos in function_info.items(): + for info in infos: + function_name = ''.join([word.capitalize() for word in info['function_name'].split("_")]) + function_route = info['function_route'] + function_method = info['function_method'] + function_params_desc = info['function_params_desc'] + + script += f""" + +@{page_name}.route('{function_route}') +class {function_name}(Resource): + @api.doc(params={function_params_desc}, operationId='{function_method.lower()}_{page_name}_{function_name}', descrption='this is a test') + def {function_method.lower()}(self): + ''' + + ''' + # Your code here + return +""" + script += f""" + +if __name__ == '__main__': + import sys + if len(sys.argv) > 1: + port = int(sys.argv[1]) + else: + port = 5000 + app.run(port=port) +""" + return script + + +if __name__ == '__main__': + file_dir = search_pages_path(Path(FATE_FLOW_HOME) / 'apps/client') + variables = read_desc_script(Path(FATE_FLOW_HOME) / 'apps/desc.py') + function_info = scan_client_app(file_dir, variables) + transfer_doc_script = generate_transfer_doc(function_info) + with open(swagger_py_file, 'w', encoding='utf-8') as file: + file.write(transfer_doc_script) diff --git a/doc/cli/checkpoint.md b/doc/cli/checkpoint.md deleted file mode 100644 index 5fb850b0c..000000000 --- a/doc/cli/checkpoint.md +++ /dev/null @@ -1,84 +0,0 @@ -## Checkpoint - -### list - -List checkpoints. - -```bash -flow checkpoint list --model-id --model-version --role --party-id --component-name -``` - -**Options** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| -------------- | ---------- | ------------------ | -------- | -------------- | -| model_id | | `--model-id` | No | Model ID | -| model_version | | `--model-version` | No | Model version | -| role | `-r` | `--role` | No | Party role | -| party_id | `-p` | `--party-id` | No | Party ID | -| component_name | `-cpn` | `--component-name` | No | Component name | - -**Example** - -```json -{ - "retcode": 0, - "retmsg": "success", - "data": [ - { - "create_time": "2021-11-07T02:34:54.683015", - "step_index": 0, - "step_name": "step_name", - "models": { - "HeteroLogisticRegressionMeta": { - "buffer_name": "LRModelMeta", - "sha1": "6871508f6e6228341b18031b3623f99a53a87147" - }, - "HeteroLogisticRegressionParam": { - "buffer_name": "LRModelParam", - "sha1": "e3cb636fc93675684bff27117943f5bfa87f3029" - } - } - } - ] -} -``` - -### get - -Get checkpoint information. - -```bash -flow checkpoint get --model-id --model-version --role --party-id --component-name --step-index -``` - - -**Example** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| -------------- | ---------- | ------------------ | -------- | ------------------------------------------- | -| model_id | | `--model-id` | No | Model ID | -| model_version | | `--model-version` | No | Model version | -| role | `-r` | `--role` | No | Party role | -| party_id | `-p` | `--party-id` | No | Party ID | -| component_name | `-cpn` | `--component-name` | No | Component name | -| step_index | | `--step-index` | Yes | Step index, cannot be used with `step_name` | -| step_name | | `--step-name` | Yes | Step name, cannot be used with `step_index` | - -**Example** - -```json -{ - "retcode": 0, - "retmsg": "success", - "data": { - "create_time": "2021-11-07T02:34:54.683015", - "step_index": 0, - "step_name": "step_name", - "models": { - "HeteroLogisticRegressionMeta": "CgJMMhEtQxzr4jYaPxkAAAAAAADwPyIHcm1zcHJvcDD///////////8BOTMzMzMzM8M/QApKBGRpZmZYAQ==", - "HeteroLogisticRegressionParam": "Ig0KAng3EW1qASu+uuO/Ig0KAng0EcNi7a65ReG/Ig0KAng4EbJbl4gvVea/Ig0KAng2EcZwlVZTkOu/Ig0KAngwEVpG8dCbGvG/Ig0KAng5ESJNTx5MLve/Ig0KAngzEZ88H9P8qfO/Ig0KAng1EVfWP8JJv/K/Ig0KAngxEVS0xVXoTem/Ig0KAngyEaApgW32Q/K/KSiiE8AukPs/MgJ4MDICeDEyAngyMgJ4MzICeDQyAng1MgJ4NjICeDcyAng4MgJ4OUj///////////8B" - } - } -} -``` diff --git a/doc/cli/checkpoint.zh.md b/doc/cli/checkpoint.zh.md deleted file mode 100644 index eaf70d7cd..000000000 --- a/doc/cli/checkpoint.zh.md +++ /dev/null @@ -1,84 +0,0 @@ -## Checkpoint - -### list - -获取 Checkpoint 模型列表。 - -```bash -flow checkpoint list --model-id --model-version --role --party-id --component-name -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| -------------- | ------ | ------------------ | -------- | ---------- | -| model_id | | `--model-id` | 否 | 模型 ID | -| model_version | | `--model-version` | 否 | 模型版本 | -| role | `-r` | `--role` | 否 | Party 角色 | -| party_id | `-p` | `--party-id` | 否 | Party ID | -| component_name | `-cpn` | `--component-name` | 否 | 组件名 | - -**样例** - -```json -{ - "retcode": 0, - "retmsg": "success", - "data": [ - { - "create_time": "2021-11-07T02:34:54.683015", - "step_index": 0, - "step_name": "step_name", - "models": { - "HeteroLogisticRegressionMeta": { - "buffer_name": "LRModelMeta", - "sha1": "6871508f6e6228341b18031b3623f99a53a87147" - }, - "HeteroLogisticRegressionParam": { - "buffer_name": "LRModelParam", - "sha1": "e3cb636fc93675684bff27117943f5bfa87f3029" - } - } - } - ] -} -``` - -### get - -获取 Checkpoint 模型信息。 - -```bash -flow checkpoint get --model-id --model-version --role --party-id --component-name --step-index -``` - - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| -------------- | ------ | ------------------ | -------- | ------------------------------------- | -| model_id | | `--model-id` | 否 | 模型 ID | -| model_version | | `--model-version` | 否 | 模型版本 | -| role | `-r` | `--role` | 否 | Party 角色 | -| party_id | `-p` | `--party-id` | 否 | Party ID | -| component_name | `-cpn` | `--component-name` | 否 | 组件名 | -| step_index | | `--step-index` | 是 | Step index，不可与 step_name 同时使用 | -| step_name | | `--step-name` | 是 | Step name，不可与 step_index 同时使用 | - -**样例** - -```json -{ - "retcode": 0, - "retmsg": "success", - "data": { - "create_time": "2021-11-07T02:34:54.683015", - "step_index": 0, - "step_name": "step_name", - "models": { - "HeteroLogisticRegressionMeta": "CgJMMhEtQxzr4jYaPxkAAAAAAADwPyIHcm1zcHJvcDD///////////8BOTMzMzMzM8M/QApKBGRpZmZYAQ==", - "HeteroLogisticRegressionParam": "Ig0KAng3EW1qASu+uuO/Ig0KAng0EcNi7a65ReG/Ig0KAng4EbJbl4gvVea/Ig0KAng2EcZwlVZTkOu/Ig0KAngwEVpG8dCbGvG/Ig0KAng5ESJNTx5MLve/Ig0KAngzEZ88H9P8qfO/Ig0KAng1EVfWP8JJv/K/Ig0KAngxEVS0xVXoTem/Ig0KAngyEaApgW32Q/K/KSiiE8AukPs/MgJ4MDICeDEyAngyMgJ4MzICeDQyAng1MgJ4NjICeDcyAng4MgJ4OUj///////////8B" - } - } -} -``` diff --git a/doc/cli/data.md b/doc/cli/data.md deleted file mode 100644 index 265f537ac..000000000 --- a/doc/cli/data.md +++ /dev/null @@ -1,282 +0,0 @@ -## Data - -### upload - -Used to upload the input data for the modeling task to the storage system supported by fate - -```bash -flow data upload -c ${conf_path} -``` - -Note: conf_path is the parameter path, the specific parameters are as follows - -**Options** - -| parameter name | required | type | description | -| :------------------ | :--- | :----------- |----------------------------------------------------------------------------------------------------------------------------------| -| file | yes | string | data storage path | -| id_delimiter | yes | string | Data separator, e.g. "," | -| head | no | int | Whether the data has a table header | yes | int -| partition | yes | int | Number of data partitions | -| storage_engine | no | string | storage engine type, default "EGGROLL", also support "HDFS", "LOCALFS", "HIVE", etc. | -| namespace | yes | string | table namespace | yes -| table_name | yes | string | table name | -| storage_address | no | object | The storage address of the corresponding storage engine is required -| use_local_data | no | int | The default is 1, which means use the data from the client's machine; 0 means use the data from the fate flow service's machine. -| drop | no | int | Whether to overwrite uploads | -| extend_sid | no | bool | Whether to add a new column for uuid id, default False | -| auto_increasing_sid | no | bool | Whether the new id column is self-increasing (will only work if extend_sid is True), default False | - -**mete information** - -| parameter name | required | type | description | -|:---------------------|:----|:-------|-------------------------------------------| -| input_format | no | string | The format of the data (danse, svmlight, tag:value), used to determine | -| delimiter | no | string | The data separator, default "," | -| tag_with_value | no | bool | Valid for tag data format, whether to carry value | -| tag_value_delimiter | no | string | tag:value data separator, default ":" | -| with_match_id | no | bool | Whether or not to carry match id | -| with_match_id | no | object | The name of the id column, effective when extend_sid is enabled, e.g., ["email", "phone"] | -| id_range | no | object | For tag/svmlight format data, which columns are ids | -| exclusive_data_type | no | string | The format of the special type data columns | -| data_type | no | string | Column data type, default "float64 | -| with_label | no | bool | Whether to have a label, default False | -| label_name | no | string | The name of the label, default "y" | -| label_type | no | string | Label type, default "int" | - -**In version 1.9.0 and later, passing in the meta parameter will generate anonymous information about the feature.** -**Example** - -- eggroll - - ```json - { - "file": "examples/data/breast_hetero_guest.csv", - "id_delimiter": ",", - "head": 1, - "partition": 10, - "namespace": "experiment", - "table_name": "breast_hetero_guest", - "storage_engine": "EGGROLL" - } - ``` - -- hdfs - - ```json - { - "file": "examples/data/breast_hetero_guest.csv", - "id_delimiter": ",", - "head": 1, - "partition": 10, - "namespace": "experiment", - "table_name": "breast_hetero_guest", - "storage_engine": "HDFS" - } - ``` - -- localfs - - ```json - { - "file": "examples/data/breast_hetero_guest.csv", - "id_delimiter": ",", - "head": 1, - "partition": 4, - "namespace": "experiment", - "table_name": "breast_hetero_guest", - "storage_engine": "LOCALFS" - } - ``` - -**return parameters** - -| parameter name | type | description | -| :------ | :----- | -------- | -| jobId | string | job id | -| retcode | int | return code | -| retmsg | string | return message | -| data | object | return data | - -**Example** - -```shell -{ - "data": { - "board_url": "http://xxx.xxx.xxx.xxx:8080/index.html#/dashboard?job_id=202111081218319075660&role=local&party_id=0", - "code": 0, - "dsl_path": "/data/projects/fate/jobs/202111081218319075660/job_dsl.json", - "job_id": "202111081218319075660", - "logs_directory": "/data/projects/fate/logs/202111081218319075660", - "message": "success", - "model_info": { - "model_id": "local-0#model", - "model_version": "202111081218319075660" - }, - "namespace": "experiment", - "pipeline_dsl_path": "/data/projects/fate/jobs/202111081218319075660/pipeline_dsl.json", - "runtime_conf_on_party_path": "/data/projects/fate/jobs/202111081218319075660/local/0/job_runtime_on_party_conf.json", - "runtime_conf_path":"/data/projects/fate/jobs/202111081218319075660/job_runtime_conf.json", - "table_name": "breast_hetero_host", - "train_runtime_conf_path":"/data/projects/fate/jobs/202111081218319075660/train_runtime_conf.json" - }, - "jobId": "202111081218319075660", - "retcode": 0, - "retmsg": "success" -} - -``` - -### upload-history - -Used to query upload table history. - -``` -flow data upload-history -l 20 -flow data upload-history --job-id $JOB_ID -``` - -**Options** - -| parameter name | required | type | description | -| :------------- | :------- | :----- | ------------------------------------------ | -| -l --limit | no | int | Number of records to return. (default: 10) | -| -j --job_id | no | string | Job ID | -| | | | | - -### download - -**Brief description:** - -Used to download data from within the fate storage engine to file format data - -```bash -flow data download -c ${conf_path} -``` - -Note: conf_path is the parameter path, the specific parameters are as follows - -**Options** - -| parameter name | required | type | description | -| :---------- | :--- | :----- | -------------- | -| output_path | yes | string | download_path | -| table_name | yes | string | fate table name | -| namespace | yes | int | fate table namespace | - -Example: - -```json -{ - "output_path": "/data/projects/fate/breast_hetero_guest.csv", - "namespace": "experiment", - "table_name": "breast_hetero_guest" -} -``` - -**return parameters** - -| parameter name | type | description | -| :------ | :----- | -------- | -| jobId | string | job id | -| retcode | int | return code | -| retmsg | string | return message | -| data | object | return data | - -**Example** - -```json -{ - "data": { - "board_url": "http://xxx.xxx.xxx.xxx:8080/index.html#/dashboard?job_id=202111081457135282090&role=local&party_id=0", - "code": 0, - "dsl_path": "/data/projects/fate/jobs/202111081457135282090/job_dsl.json", - "job_id": "202111081457135282090", - "logs_directory": "/data/projects/fate/logs/202111081457135282090", - "message": "success", - "model_info": { - "model_id": "local-0#model", - "model_version": "202111081457135282090" - }, - "pipeline_dsl_path": "/data/projects/fate/jobs/202111081457135282090/pipeline_dsl.json", - "runtime_conf_on_party_path": "/data/projects/fate/jobs/202111081457135282090/local/0/job_runtime_on_party_conf.json", - "runtime_conf_path": "/data/projects/fate/jobs/202111081457135282090/job_runtime_conf.json", - "train_runtime_conf_path": "/data/projects/fate/jobs/202111081457135282090/train_runtime_conf.json" - }, - "jobId": "202111081457135282090", - "retcode": 0, - "retmsg": "success" -} - -``` - -### writer - -**Brief description:** - -Used to download data from the fate storage engine to the external engine or to save data as a new table - -```bash -flow data writer -c ${conf_path} -``` - -Note: conf_path is the parameter path, the specific parameters are as follows - -**Options** - -| parameter name | required | type | description | -| :---------- | :--- | :----- | -------------- | -| table_name | yes | string | fate table name | -| namespace | yes | int | fate table namespace | -| storage_engine | no | string | Storage type, e.g., MYSQL | -| address | no | object | storage_address | -| output_namespace | no | string | Save as a table namespace for fate | -| output_name | no | string | Save as fate's table name | -**Note: storage_engine, address are combined parameters that provide storage to the specified engine. -output_namespace, output_name are also combined parameters, providing the function to save as a new table of the same engine** - -Example: - -```json -{ - "table_name": "name1", - "namespace": "namespace1", - "output_name": "name2", - "output_namespace": "namespace2" -} -``` - -**return** - -| parameter name | type | description | -| :------ | :----- | -------- | -| jobId | string | job id | -| retcode | int | return code | -| retmsg | string | return information | -| data | object | return data | - -**Example** - -```json -{ - "data": { - "board_url": "http://xxx.xxx.xxx.xxx:8080/index.html#/dashboard?job_id=202201121235115028490&role=local&party_id=0", - "code": 0, - "dsl_path": "/data/projects/fate/fateflow/jobs/202201121235115028490/job_dsl.json", - "job_id": "202201121235115028490", - "logs_directory": "/data/projects/fate/fateflow/logs/202201121235115028490", - "message": "success", - "model_info": { - "model_id": "local-0#model", - "model_version": "202201121235115028490" - }, - "pipeline_dsl_path": "/data/projects/fate/fateflow/jobs/202201121235115028490/pipeline_dsl.json", - "runtime_conf_on_party_path": "/data/projects/fate/fateflow/jobs/202201121235115028490/local/0/job_runtime_on_party_conf.json", - "runtime_conf_path":"/data/projects/fate/fateflow/jobs/202201121235115028490/job_runtime_conf.json", - "train_runtime_conf_path": "/data/projects/fate/fateflow/jobs/202201121235115028490/train_runtime_conf.json" - }, - "jobId": "202201121235115028490", - "retcode": 0, - "retmsg": "success" -} -``` diff --git a/doc/cli/data.zh.md b/doc/cli/data.zh.md deleted file mode 100644 index d220f06cc..000000000 --- a/doc/cli/data.zh.md +++ /dev/null @@ -1,285 +0,0 @@ -## Data - -### upload - -用于上传建模任务的输入数据到fate所支持的存储系统 - -```bash -flow data upload -c ${conf_path} -``` - -注: conf_path为参数路径，具体参数如下 - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -|:--------------------|:----|:-------|-------------------------------------------------| -| file | 是 | string | 数据存储路径 | -| id_delimiter | 是 | string | 数据分隔符,如"," | -| head | 否 | int | 数据是否有表头 | -| partition | 是 | int | 数据分区数 | -| storage_engine | 否 | string | 存储引擎类型，默认"EGGROLL"，还支持"HDFS","LOCALFS", "HIVE"等 | -| namespace | 是 | string | 表命名空间 | -| table_name | 是 | string | 表名 | -| storage_address | 否 | object | 需要填写对应存储引擎的存储地址 | -| use_local_data | 否 | int | 默认1，代表使用client机器的数据;0代表使用fate flow服务所在机器的数据 | -| drop | 否 | int | 是否覆盖上传 | -| extend_sid | 否 | bool | 是否新增一列uuid id，默认False | -| auto_increasing_sid | 否 | bool | 新增的id列是否自增(extend_sid为True才会生效), 默认False | -| with_meta | 否 | bool | 是否携带meta数据, 默认False | -| meta | 否 | object | 元数据, 默认为空，with_meta为true生效 | - -**mete信息** - -| 参数名 | 必选 | 类型 | 说明 | -|:---------------------|:----|:-------|-------------------------------------------| -| input_format | 否 | string | 数据格式(danse、svmlight、tag:value)，用来判断 | -| delimiter | 否 | string | 数据分隔符,默认"," | -| tag_with_value | 否 | bool | 对tag的数据格式生效，是否携带value | -| tag_value_delimiter | 否 | string | tag:value数据分隔符，默认":" | -| with_match_id | 否 | bool | 是否携带match id | -| id_list | 否 | object | id列名称，开启extend_sid下生效，如：["imei", "phone"] | -| id_range | 否 | object | 对于tag/svmlight格式数据，哪几列为id | -| exclusive_data_type | 否 | string | 特殊类型数据列格式 | -| data_type | 否 | string | 列数据类型，默认"float64 | -| with_label | 否 | bool | 是否有标签，默认False | -| label_name | 否 | string | 标签名，默认"y" | -| label_type | 否 | string | 标签类型, 默认"int" | - -**注意：在1.9.0及之后的版本中，若传入meta参数，会生成特征的匿名信息。** - -**样例** - -- eggroll - - ```json - { - "file": "examples/data/breast_hetero_guest.csv", - "id_delimiter": ",", - "head": 1, - "partition": 10, - "namespace": "experiment", - "table_name": "breast_hetero_guest", - "storage_engine": "EGGROLL" - } - ``` - -- hdfs - - ```json - { - "file": "examples/data/breast_hetero_guest.csv", - "id_delimiter": ",", - "head": 1, - "partition": 10, - "namespace": "experiment", - "table_name": "breast_hetero_guest", - "storage_engine": "HDFS" - } - ``` - -- localfs - - ```json - { - "file": "examples/data/breast_hetero_guest.csv", - "id_delimiter": ",", - "head": 1, - "partition": 4, - "namespace": "experiment", - "table_name": "breast_hetero_guest", - "storage_engine": "LOCALFS" - } - ``` - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| jobId | string | 任务id | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -**样例** - -```shell -{ - "data": { - "board_url": "http://xxx.xxx.xxx.xxx:8080/index.html#/dashboard?job_id=202111081218319075660&role=local&party_id=0", - "code": 0, - "dsl_path": "/data/projects/fate/jobs/202111081218319075660/job_dsl.json", - "job_id": "202111081218319075660", - "logs_directory": "/data/projects/fate/logs/202111081218319075660", - "message": "success", - "model_info": { - "model_id": "local-0#model", - "model_version": "202111081218319075660" - }, - "namespace": "experiment", - "pipeline_dsl_path": "/data/projects/fate/jobs/202111081218319075660/pipeline_dsl.json", - "runtime_conf_on_party_path": "/data/projects/fate/jobs/202111081218319075660/local/0/job_runtime_on_party_conf.json", - "runtime_conf_path": "/data/projects/fate/jobs/202111081218319075660/job_runtime_conf.json", - "table_name": "breast_hetero_host", - "train_runtime_conf_path": "/data/projects/fate/jobs/202111081218319075660/train_runtime_conf.json" - }, - "jobId": "202111081218319075660", - "retcode": 0, - "retmsg": "success" -} - -``` - -### upload-history - -用于查询上传历史 - -``` -flow data upload-history -l 20 -flow data upload-history --job-id $JOB_ID -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :---------- | :--- | :----- | ------------------- | -| -l --limit | no | int | 返回数量 (默认: 10) | -| -j --job_id | no | string | 任务ID | -| | | | | - -### download - -**简要描述：** - -用于下载fate存储引擎内的数据到文件格式数据 - -```bash -flow data download -c ${conf_path} -``` - -注: conf_path为参数路径，具体参数如下 - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :---------- | :--- | :----- | -------------- | -| output_path | 是 | string | 下载路径 | -| table_name | 是 | string | fate表名 | -| namespace | 是 | int | fate表命名空间 | - -样例: - -```json -{ - "output_path": "/data/projects/fate/breast_hetero_guest.csv", - "namespace": "experiment", - "table_name": "breast_hetero_guest" -} -``` - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| jobId | string | 任务id | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -样例 - -```json -{ - "data": { - "board_url": "http://xxx.xxx.xxx.xxx:8080/index.html#/dashboard?job_id=202111081457135282090&role=local&party_id=0", - "code": 0, - "dsl_path": "/data/projects/fate/jobs/202111081457135282090/job_dsl.json", - "job_id": "202111081457135282090", - "logs_directory": "/data/projects/fate/logs/202111081457135282090", - "message": "success", - "model_info": { - "model_id": "local-0#model", - "model_version": "202111081457135282090" - }, - "pipeline_dsl_path": "/data/projects/fate/jobs/202111081457135282090/pipeline_dsl.json", - "runtime_conf_on_party_path": "/data/projects/fate/jobs/202111081457135282090/local/0/job_runtime_on_party_conf.json", - "runtime_conf_path": "/data/projects/fate/jobs/202111081457135282090/job_runtime_conf.json", - "train_runtime_conf_path": "/data/projects/fate/jobs/202111081457135282090/train_runtime_conf.json" - }, - "jobId": "202111081457135282090", - "retcode": 0, - "retmsg": "success" -} - -``` - -### writer - -**简要描述：** - -用于下载fate存储引擎内的数据到外部引擎或者将数据另存为新表 - -```bash -flow data writer -c ${conf_path} -``` - -注: conf_path为参数路径，具体参数如下 - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :---------- | :--- | :----- | -------------- | -| table_name | 是 | string | fate表名 | -| namespace | 是 | int | fate表命名空间 | -| storage_engine | 否 | string | 存储类型,如：MYSQL | -| address | 否 | object | 存储地址 | -| output_namespace | 否 | string | 另存为fate的表命名空间 | -| output_name | 否 | string | 另存为fate的表名 | -**注: storage_engine、address是组合参数，提供存储到指定引擎的功能； -output_namespace、output_name也是组合参数，提供另存为同种引擎的新表功能** - -样例: - -```json -{ - "table_name": "name1", - "namespace": "namespace1", - "output_name": "name2", - "output_namespace": "namespace2" -} -``` - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| jobId | string | 任务id | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -样例 - -```json -{ - "data": { - "board_url": "http://xxx.xxx.xxx.xxx:8080/index.html#/dashboard?job_id=202201121235115028490&role=local&party_id=0", - "code": 0, - "dsl_path": "/data/projects/fate/fateflow/jobs/202201121235115028490/job_dsl.json", - "job_id": "202201121235115028490", - "logs_directory": "/data/projects/fate/fateflow/logs/202201121235115028490", - "message": "success", - "model_info": { - "model_id": "local-0#model", - "model_version": "202201121235115028490" - }, - "pipeline_dsl_path": "/data/projects/fate/fateflow/jobs/202201121235115028490/pipeline_dsl.json", - "runtime_conf_on_party_path": "/data/projects/fate/fateflow/jobs/202201121235115028490/local/0/job_runtime_on_party_conf.json", - "runtime_conf_path": "/data/projects/fate/fateflow/jobs/202201121235115028490/job_runtime_conf.json", - "train_runtime_conf_path": "/data/projects/fate/fateflow/jobs/202201121235115028490/train_runtime_conf.json" - }, - "jobId": "202201121235115028490", - "retcode": 0, - "retmsg": "success" -} -``` diff --git a/doc/cli/job.md b/doc/cli/job.md deleted file mode 100644 index c355a0aa0..000000000 --- a/doc/cli/job.md +++ /dev/null @@ -1,277 +0,0 @@ -## Job - -### submit - -Build a federated learning job with two configuration files: job dsl and job conf, and submit it to the scheduler for execution - -```bash -flow job submit [options] -``` - -**Options** - -| parameter name | required | type | description | -| :-------------- | :------- | :----- | --------------- | -| -d, --dsl-path | yes | string | path to job dsl | -| -c, --conf-path | yes | string | job conf's path | - -**Returns** - -| parameter name | type | description | -| :------------------------------ | :----- | --------------------------------------------------------------------------------------------------------------------- | -| retcode | int | return code | -| retmsg | string | return message | -| jobId | string | Job ID | -| data | dict | return data | -| data.dsl_path | string | The path to the actual running dsl configuration generated by the system based on the submitted dsl content | -| data.runtime_conf_on_party_path | string | The system-generated path to the actual running conf configuration for each party based on the submitted conf content | -| data.dsl_path | string | The system-generated path to the actual running conf configuration for each party based on the submitted conf content | -| data.board_url | string | fateboard view address | -| data.model_info | dict | Model identification information | - -**Example** - -```json -{ - "data": { - "board_url": "http://127.0.0.1:8080/index.html#/dashboard?job_id=202111061608424372620&role=guest&party_id=9999", - "code": 0, - "dsl_path": "$FATE_PROJECT_BASE/jobs/202111061608424372620/job_dsl.json", - "job_id": "202111061608424372620", - "logs_directory": "$FATE_PROJECT_BASE/logs/202111061608424372620", - "message": "success", - "model_info": { - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "202111061608424372620" - }, - "pipeline_dsl_path": "$FATE_PROJECT_BASE/jobs/202111061608424372620/pipeline_dsl.json", - "runtime_conf_on_party_path": "$FATE_FATE_PROJECT_BASE/jobs/202111061608424372620/guest/9999/job_runtime_on_party_conf.json", - "runtime_conf_path":"$FATE_PROJECT_BASE/jobs/202111061608424372620/job_runtime_conf.json", - "train_runtime_conf_path": "$FATE_PROJECT_BASE/jobs/202111061608424372620/train_runtime_conf.json" - }, - "jobId": "202111061608424372620", - "retcode": 0, - "retmsg": "success" -} -``` - -### rerun - -Rerun a job - -```bash -flow job rerun [options] -``` - -**Options** - -| parameter name | required | type | description | -| :------------- | :------- | :--- | ----------- |------- | -| -j, --job-id | yes | string | job id path | -| --cpn, --component-name | no | string | Specifies which component to rerun from, unspecified components will not be executed if they have no upstream dependencies on the specified component; if not specified, the entire job will be rerun | -| --force | no | bool | The job will be rerun even if it succeeds; if not specified, the job will be skipped if it succeeds | - -**Returns** - -| parameter name | type | description | -| :------------- | :----- | ------------------ | -| retcode | int | return code | -| retmsg | string | return message | -| jobId | string | Job ID | -| data | dict | return data | - -**Example** - -```bash -flow job rerun -j 202111031100369723120 -``` - -```bash -flow job rerun -j 202111031100369723120 -cpn hetero_lr_0 -``` - -```bash -flow job rerun -j 202111031100369723120 -cpn hetero_lr_0 --force -``` - -### parameter-update - -Update the job parameters - -```bash -flow job parameter-update [options] -``` - -**Options** - -| parameter-name | required | type | description | -| :-------------- | :------- | :----- | ------------------------------------------------------------------------------------------------------------------ | -| -j, --job-id | yes | string | job id path | -| -c, --conf-path | yes | string | The contents of the job conf that needs to be updated, no need to fill in parameters that don't need to be updated | - -**Returns** - -| parameter name | type | description | -| :------------- | :----- | ---------------------------- | -| retcode | int | return code | -| retmsg | string | return message | -| jobId | string | Job ID | -| data | dict | Returns the updated job conf | - -**Example** - -Assuming that the job is updated with some of the execution parameters of the hetero_lr_0 component, the configuration file is as follows. -```bash -{ - "job_parameters": { - }, - "component_parameters": { - "common": { - "hetero_lr_0": { - "alpha": 0.02, - "max_iter": 5 - } - } - } -} -``` - -Execution of the following command takes effect. - -```bash -flow job parameter-update -j 202111061957421943730 -c examples/other/update_parameters.json -``` - -Execute the following command to rerun. - -```bash -flow job rerun -j 202111061957421943730 -cpn hetero_lr_0 --force -``` - -### stop - -Cancels or terminates the specified job - -**Options** - -| number | parameters | short format | long format | required parameters | parameter description | -| ------ | ---------- | ------------ | ----------- | ------------------- | --------------------- | -| 1 | job_id | `-j` | `--job_id` | yes | Job ID | - -**Example** - -``` bash -flow job stop -j $JOB_ID -``` - -### query - -Retrieve task information. -**Options** - -| number | parameters | short-format | long-format | required parameters | parameter description | -| ------ | ---------- | ------------ | ------------ | ------------------- | --------------------- | -| 1 | job_id | `-j` | `--job_id` | no | Job ID | -| 2 | role | `-r` | `--role` | no | role | -| 3 | party_id | `-p` | `--party_id` | no | Party ID | -| 4 | status | `-s` | `--status` | No | Task status | - -**Example** - -``` bash -flow job query -r guest -p 9999 -s complete -flow job query -j $JOB_ID -``` - -### view - -Retrieve the job data view. -**Options** - -| number | parameters | short-format | long-format | required parameters | parameter description | -| ------ | ---------- | ------------ | ------------ | ------------------- | --------------------- | -| 1 | job_id | `-j` | `--job_id` | yes | Job ID | -| 2 | role | `-r` | `--role` | no | role | -| 3 | party_id | `-p` | `--party_id` | no | Party ID | -| 4 | status | `-s` | `--status` | No | Task status | - -**Example** - -``` bash -flow job view -j $JOB_ID -s complete -``` - -### config - -Download the configuration file for the specified job to the specified directory. - -**Options** - -| number | parameters | short-format | long-format | required parameters | parameter description | -| ------ | ----------- | ------------ | --------------- | ------------------- | --------------------- | -| 1 | job_id | `-j` | `--job_id` | yes | Job ID | -| 2 | role | `-r` | `--role` | yes | role | -| 3 | party_id | `-p` | `--party_id` | yes | Party ID | -| 4 | output_path | `-o` | `--output-path` | yes | output directory | - -**Example** - -``` bash -flow job config -j $JOB_ID -r host -p 10000 --output-path . /examples/ -``` - -### log - -Download the log file of the specified job to the specified directory. -**Options** - -| number | parameters | short-format | long-format | required parameters | parameter description | -| ------ | ----------- | ------------ | --------------- | ------------------- | --------------------- | -| 1 | job_id | `-j` | `--job_id` | yes | Job ID | -| 2 | output_path | `-o` | `--output-path` | yes | output directory | - -**Example** - -``` bash -flow job log -j JOB_ID --output-path . /examples/ -``` - -### list - -Show the list of jobs. -**Options** - -| number | parameters | short-format | long-format | required parameters | parameter description | -| ------ | ---------- | ------------ | ----------- | ------------------- | -------------------------------------- | -| 1 | limit | `-l` | `-limit` | no | Returns the number limit (default: 10) | - -**Example** - -``` bash -flow job list -flow job list -l 30 -``` - -### dsl - -Predictive DSL file generator. -**Options** - -| number | parameters | short-format | long-format | required parameters | parameter description | -| ------ | -------------- | ------------ | ----------------- | ------------------- | ------------------------------------------------------------ | -| 1 | cpn_list | | `-cpn-list` | No | List of user-specified component names | -| 2 | cpn_path | | `-cpn-path` | No | User-specified path to a file with a list of component names | -| 3 | train_dsl_path | | `-train-dsl-path` | yes | path to the training dsl file | -| 4 | output_path | `-o` | `--output-path` | no | output directory path | - -**Example** - -``` bash -flow job dsl --cpn-path fate_flow/examples/component_list.txt --train-dsl-path fate_flow/examples/test_hetero_lr_job_dsl.json - -flow job dsl --cpn-path fate_flow/examples/component_list.txt --train-dsl-path fate_flow/examples/test_hetero_lr_job_dsl.json -o fate_flow /examples/ - -flow job dsl --cpn-list "dataio_0, hetero_feature_binning_0, hetero_feature_selection_0, evaluation_0" --train-dsl-path fate_flow/examples/ test_hetero_lr_job_dsl.json -o fate_flow/examples/ - -flow job dsl --cpn-list [dataio_0,hetero_feature_binning_0,hetero_feature_selection_0,evaluation_0] --train-dsl-path fate_flow/examples/ test_hetero_lr_job_dsl.json -o fate_flow/examples/ -``` diff --git a/doc/cli/job.zh.md b/doc/cli/job.zh.md deleted file mode 100644 index a54e5f3b7..000000000 --- a/doc/cli/job.zh.md +++ /dev/null @@ -1,275 +0,0 @@ -## Job - -### submit - -通过两个配置文件：job dsl和job conf构建一个联邦学习作业，提交到调度系统执行 - -```bash -flow job submit [options] -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :-------------- | :--- | :----- | -------------- | -| -d, --dsl-path | 是 | string | job dsl的路径 | -| -c, --conf-path | 是 | string | job conf的路径 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------------------------------ | :----- | --------------------------------------------------------------------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| jobId | string | 作业ID | -| data | dict | 返回数据 | -| data.dsl_path | string | 依据提交的dsl内容，由系统生成的实际运行dsl配置的存放路径 | -| data.runtime_conf_on_party_path | string | 依据提交的conf内容，由系统生成的在每个party实际运行conf配置的存放路径 | -| data.board_url | string | fateboard查看地址 | -| data.model_info | dict | 模型标识信息 | - -**样例** - -```json -{ - "data": { - "board_url": "http://127.0.0.1:8080/index.html#/dashboard?job_id=202111061608424372620&role=guest&party_id=9999", - "code": 0, - "dsl_path": "$FATE_PROJECT_BASE/jobs/202111061608424372620/job_dsl.json", - "job_id": "202111061608424372620", - "logs_directory": "$FATE_PROJECT_BASE/logs/202111061608424372620", - "message": "success", - "model_info": { - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "202111061608424372620" - }, - "pipeline_dsl_path": "$FATE_PROJECT_BASE/jobs/202111061608424372620/pipeline_dsl.json", - "runtime_conf_on_party_path": "$FATE_FATE_PROJECT_BASE/jobs/202111061608424372620/guest/9999/job_runtime_on_party_conf.json", - "runtime_conf_path": "$FATE_PROJECT_BASE/jobs/202111061608424372620/job_runtime_conf.json", - "train_runtime_conf_path": "$FATE_PROJECT_BASE/jobs/202111061608424372620/train_runtime_conf.json" - }, - "jobId": "202111061608424372620", - "retcode": 0, - "retmsg": "success" -} -``` - -### rerun - -重新运行某个作业 - -```bash -flow job rerun [options] -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :--------------------- | :--- | :----- | ----------------------------------------------------------------------------------------------------- | -| -j, --job-id | 是 | string | job id 路径 | -| -cpn, --component-name | 否 | string | 指定从哪个组件重跑，没被指定的组件若与指定组件没有上游依赖关系则不会执行;若不指定该参数则整个作业重跑 | -| --force | 否 | bool | 作业即使成功也重跑;若不指定该参数，作业如果成功，则跳过重跑 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| jobId | string | 作业ID | -| data | dict | 返回数据 | - -**样例** - -```bash -flow job rerun -j 202111031100369723120 -``` - -```bash -flow job rerun -j 202111031100369723120 -cpn hetero_lr_0 -``` - -```bash -flow job rerun -j 202111031100369723120 -cpn hetero_lr_0 --force -``` - -### parameter-update - -更新作业参数 - -```bash -flow job parameter-update [options] -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :-------------- | :--- | :----- | ---------------------------------------------------- | -| -j, --job-id | 是 | string | job id 路径 | -| -c, --conf-path | 是 | string | 需要更新的job conf的内容，不需要更新的参数不需要填写 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------------------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| jobId | string | 作业ID | -| data | dict | 返回更新后的job conf | - -**样例** - -假设更新job中hetero_lr_0这个组件的部分执行参数，配置文件如下： -```bash -{ - "job_parameters": { - }, - "component_parameters": { - "common": { - "hetero_lr_0": { - "alpha": 0.02, - "max_iter": 5 - } - } - } -} -``` - -执行如下命令生效： - -```bash -flow job parameter-update -j 202111061957421943730 -c examples/other/update_parameters.json -``` - -执行如下命令重跑： - -```bash -flow job rerun -j 202111061957421943730 -cpn hetero_lr_0 --force -``` - -### stop - -取消或终止指定任务 - -**选项** - -| 编号 | 参数 | 短格式 | 长格式 | 必要参数 | 参数介绍 | -| ---- | ------ | ------ | ---------- | -------- | -------- | -| 1 | job_id | `-j` | `--job_id` | 是 | Job ID | - -**样例** - -``` bash -flow job stop -j $JOB_ID -``` - -### query - -检索任务信息。 -**选项** - -| 编号 | 参数 | 短格式 | 长格式 | 必要参数 | 参数介绍 | -| ---- | -------- | ------ | ------------ | -------- | -------- | -| 1 | job_id | `-j` | `--job_id` | 否 | Job ID | -| 2 | role | `-r` | `--role` | 否 | 角色 | -| 3 | party_id | `-p` | `--party_id` | 否 | Party ID | -| 4 | status | `-s` | `--status` | 否 | 任务状态 | - -**样例**： - -``` bash -flow job query -r guest -p 9999 -s complete -flow job query -j $JOB_ID -``` - -### view - -检索任务数据视图。 -**选项** - -| 编号 | 参数 | 短格式 | 长格式 | 必要参数 | 参数介绍 | -| ---- | -------- | ------ | ------------ | -------- | -------- | -| 1 | job_id | `-j` | `--job_id` | 是 | Job ID | -| 2 | role | `-r` | `--role` | 否 | 角色 | -| 3 | party_id | `-p` | `--party_id` | 否 | Party ID | -| 4 | status | `-s` | `--status` | 否 | 任务状态 | - -**样例**： - -``` bash -flow job view -j $JOB_ID -s complete -``` - -### config - -下载指定任务的配置文件到指定目录。 -**选项** - -| 编号 | 参数 | 短格式 | 长格式 | 必要参数 | 参数介绍 | -| ---- | ----------- | ------ | --------------- | -------- | -------- | -| 1 | job_id | `-j` | `--job_id` | 是 | Job ID | -| 2 | role | `-r` | `--role` | 是 | 角色 | -| 3 | party_id | `-p` | `--party_id` | 是 | Party ID | -| 4 | output_path | `-o` | `--output-path` | 是 | 输出目录 | - -**样例**： - -``` bash -flow job config -j $JOB_ID -r host -p 10000 --output-path ./examples/ -``` - -### log - -下载指定任务的日志文件到指定目录。 -**选项** - -| 编号 | 参数 | 短格式 | 长格式 | 必要参数 | 参数介绍 | -| ---- | ----------- | ------ | --------------- | -------- | -------- | -| 1 | job_id | `-j` | `--job_id` | 是 | Job ID | -| 2 | output_path | `-o` | `--output-path` | 是 | 输出目录 | - -**样例**： - -``` bash -flow job log -j JOB_ID --output-path ./examples/ -``` - -### list - -展示任务列表。 -**选项** - -| 编号 | 参数 | 短格式 | 长格式 | 必要参数 | 参数介绍 | -| ---- | ----- | ------ | --------- | -------- | ------------------------ | -| 1 | limit | `-l` | `--limit` | 否 | 返回数量限制（默认：10） | - -**样例**： - -``` bash -flow job list -flow job list -l 30 -``` - -### dsl - -预测DSL文件生成器。 -**选项** - -| 编号 | 参数 | 短格式 | 长格式 | 必要参数 | 参数介绍 | -| ---- | -------------- | ------ | ------------------ | -------- | -------------------------------- | -| 1 | cpn_list | | `--cpn-list` | 否 | 用户指定组件名列表 | -| 2 | cpn_path | | `--cpn-path` | 否 | 用户指定带有组件名列表的文件路径 | -| 3 | train_dsl_path | | `--train-dsl-path` | 是 | 训练dsl文件路径 | -| 4 | output_path | `-o` | `--output-path` | 否 | 输出目录路径 | - -**样例**： - -``` bash -flow job dsl --cpn-path fate_flow/examples/component_list.txt --train-dsl-path fate_flow/examples/test_hetero_lr_job_dsl.json - -flow job dsl --cpn-path fate_flow/examples/component_list.txt --train-dsl-path fate_flow/examples/test_hetero_lr_job_dsl.json -o fate_flow/examples/ - -flow job dsl --cpn-list "dataio_0, hetero_feature_binning_0, hetero_feature_selection_0, evaluation_0" --train-dsl-path fate_flow/examples/test_hetero_lr_job_dsl.json -o fate_flow/examples/ - -flow job dsl --cpn-list [dataio_0,hetero_feature_binning_0,hetero_feature_selection_0,evaluation_0] --train-dsl-path fate_flow/examples/test_hetero_lr_job_dsl.json -o fate_flow/examples/ -``` diff --git a/doc/cli/key.md b/doc/cli/key.md deleted file mode 100644 index 5f6dda6e1..000000000 --- a/doc/cli/key.md +++ /dev/null @@ -1,101 +0,0 @@ -## Key - -### query - -Query the public key information of our or partner's fate site - -```bash -flow key query -p 9999 -``` -**Options** - -| parameters | short-format | long-format | required | type | description | -| :-------- | :-----| :-----| :-----| :-----| -------------- | -| party_id | `-p` | `--party-id` | yes | string | site id | - -**returns** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return-code | -| retmsg | string | return information | -| data | object | return data | - -Sample - -```json -{ - "data": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzxgbxa3cfhvwbu0AFfY/\ nkm7uFZ17J0EEDgaIWlrLakds7XboU5iOT0eReQp/KG3R0fVM9rBtdj8NcBcArtZ9\n2242Atls3jiuza/MPPo9XACnedGW7O+ VAfvVmq2sdmKZMX5l7krEXYN645UZAd8b\nhIh+xf0qGW6IgxyKvqF13VxxB7OMUzUwyY/ZcN2rW1urfdXsCNoQ1cFl3KaarkHl\nn/ gBMcCDvACXoKysFnFE7L4E7CGglYaDBJrfIyti+sbSVNxUDx2at2VXqj/PohTa\nkBKfrgK7sT85gz1sc9uRwhwF4nOY7izq367S7t/W8BJ75gWsr+lhhiIfE19RBbBQ\n /wIDAQAB\n-----END PUBLIC KEY-----", - "retcode": 0, - "retmsg": "success" -} -``` - -### save - -Used to save other fate site public key information, that is, for cooperation with other sites - -```bash -flow key save -c fateflow/examples/key/save_public_key.json -``` - -**Options** - -| parameters | short format | long format | required | type | description | -| :-------- | :-----| :-----| :-----| :----- | -------------- | -| conf_path | `-c` | `-conf-path` | yes | string | configuration-path | - -Note: conf_path is the parameter path, the specific parameters are as follows - -| parameter name | required | type | description | -|:---------------| :--- | :----- |---------------------------------| -| party_id | yes | string | site id | -| key | yes | string | site public key | - -**return** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | - - -Sample - -```json -{ - "retcode": 0, - "retmsg": "success" -} -``` - -### delete - -Delete the partner site public key, i.e. cancel the partnership - -```bash -flow key delete -p 9999 -``` - -**Options** - -| parameters | short-format | long-format | required | type | description | -| :------ | :----- | :-----| :-----| :-----| -------- | -| party_id | `-p` | `--party-id` | yes | string | site id | - -**returns** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return-code | -| retmsg | string | return message | - - -Sample - -```json -{ - "retcode": 0, - "retmsg": "success" -} -``` \ No newline at end of file diff --git a/doc/cli/key.zh.md b/doc/cli/key.zh.md deleted file mode 100644 index e6b3c83d1..000000000 --- a/doc/cli/key.zh.md +++ /dev/null @@ -1,101 +0,0 @@ -## Key - -### query - -用于查询本方或合作方fate站点公钥信息 - -```bash -flow key query -p 9999 -``` -**选项** - -| 参数 | 短格式 | 长格式 | 必选 | 类型 | 说明 | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| party_id | `-p` | `--party-id` |是 | string | 站点id | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -样例 - -```json -{ - "data": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzxgbxa3cfhvwbu0AFfY/\nkm7uFZ17J0EEDgaIWlrLakds7XboU5iOT0eReQp/KG3R0fVM9rBtdj8NcBcArtZ9\n2242Atls3jiuza/MPPo9XACnedGW7O+VAfvVmq2sdmKZMX5l7krEXYN645UZAd8b\nhIh+xf0qGW6IgxyKvqF13VxxB7OMUzUwyY/ZcN2rW1urfdXsCNoQ1cFl3KaarkHl\nn/gBMcCDvACXoKysFnFE7L4E7CGglYaDBJrfIyti+sbSVNxUDx2at2VXqj/PohTa\nkBKfrgK7sT85gz1sc9uRwhwF4nOY7izq367S7t/W8BJ75gWsr+lhhiIfE19RBbBQ\n/wIDAQAB\n-----END PUBLIC KEY-----", - "retcode": 0, - "retmsg": "success" -} -``` - -### save - -用于保存其它fate站点公钥信息，即为和其他站点合作 - -```bash -flow key save -c fateflow/examples/key/save_public_key.json -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 必选 | 类型 | 说明 | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| conf_path | `-c` |`--conf-path` |是 | string | 配置路径 | - -注: conf_path为参数路径，具体参数如下 - -| 参数名 | 必选 | 类型 | 说明 | -|:---------------| :--- | :----- |---------------------------------| -| party_id | 是 | string | 站点id | -| key | 是 | string | 站点公钥 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | - - -样例 - -```json -{ - "retcode": 0, - "retmsg": "success" -} -``` - -### delete - -删除合作方站点公钥，即为取消合作关系 - -```bash -flow key delete -p 9999 -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 必选 | 类型 | 说明 | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| party_id | `-p` | `--party-id` |是 | string | 站点id | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | - - -样例 - -```json -{ - "retcode": 0, - "retmsg": "success" -} -``` diff --git a/doc/cli/model.md b/doc/cli/model.md deleted file mode 100644 index 0c51cb332..000000000 --- a/doc/cli/model.md +++ /dev/null @@ -1,376 +0,0 @@ -## Model - -### load - -Load a model generated by `deploy` to Fate-Serving. - - -```bash -flow model load -c examples/model/publish_load_model.json -flow model load -j -``` - -**Options** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| --------- | ---------- | ------------- | -------- | ---------------- | -| conf_path | `-c` | `--conf-path` | Yes | Config file path | -| job_id | `-j` | `--job-id` | Yes | Job ID | - -**Example** - -```json -{ - "data": { - "detail": { - "guest": { - "9999": { - "retcode": 0, - "retmsg": "success" - } - }, - "host": { - "10000": { - "retcode": 0, - "retmsg": "success" - } - } - }, - "guest": { - "9999": 0 - }, - "host": { - "10000": 0 - } - }, - "jobId": "202111091122168817080", - "retcode": 0, - "retmsg": "success" -} -``` - -### bind - -Bind a model generated by `deploy` to Fate-Serving. - -```bash -flow model bind -c examples/model/bind_model_service.json -flow model bind -c examples/model/bind_model_service.json -j -``` - -**Options** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| --------- | ---------- | ------------- | -------- | ---------------- | -| conf_path | `-c` | `--conf-path` | No | Config file path | -| job_id | `-j` | `--job-id` | Yes | Job ID | - -**Example** - -```json -{ - "retcode": 0, - "retmsg": "service id is 123" -} -``` - -### import - -Import the model from a file or storage engine. - -```bash -flow model import -c examples/model/import_model.json -flow model import -c examples/model/restore_model.json --from-database -``` - -**Options** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| ------------- | ---------- | ----------------- | -------- | ------------------------------------ | -| conf_path | `-c` | `--conf-path` | No | Config file path | -| from_database | | `--from-database` | Yes | Import the model from storage engine | - -**Example** - -```json -{ - "data": { - "job_id": "202208261102212849780", - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "foobar", - "party_id": "9999", - "role": "guest" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### export - -Export the model to a file or storage engine. - -```bash -flow model export -c examples/model/export_model.json -flow model export -c examples/model/store_model.json --to-database -``` - -**Options** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| ----------- | ---------- | --------------- | -------- | ---------------------------------- | -| conf_path | `-c` | `--conf-path` | No | Config file path | -| to_database | | `--to-database` | Yes | Export the model to storage engine | - -**Example** - -```json -{ - "data": { - "board_url": "http://127.0.0.1:8080/index.html#/dashboard?job_id=202111091124582110490&role=local&party_id=0", - "code": 0, - "dsl_path": "/root/Codes/FATE-Flow/jobs/202111091124582110490/job_dsl.json", - "job_id": "202111091124582110490", - "logs_directory": "/root/Codes/FATE-Flow/logs/202111091124582110490", - "message": "success", - "model_info": { - "model_id": "local-0#model", - "model_version": "202111091124582110490" - }, - "pipeline_dsl_path": "/root/Codes/FATE-Flow/jobs/202111091124582110490/pipeline_dsl.json", - "runtime_conf_on_party_path": "/root/Codes/FATE-Flow/jobs/202111091124582110490/local/0/job_runtime_on_party_conf.json", - "runtime_conf_path": "/root/Codes/FATE-Flow/jobs/202111091124582110490/job_runtime_conf.json", - "train_runtime_conf_path": "/root/Codes/FATE-Flow/jobs/202111091124582110490/train_runtime_conf.json" - }, - "jobId": "202111091124582110490", - "retcode": 0, - "retmsg": "success" -} -``` - -### migrate - -Migrate the model. - -```bash -flow model migrate -c examples/model/migrate_model.json -``` - -**Options** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| --------- | ---------- | ------------- | -------- | ---------------- | -| conf_path | `-c` | `--conf-path` | No | Config file path | - -**Example** - -```json -{ - "data": { - "arbiter": { - "10000": 0 - }, - "detail": { - "arbiter": { - "10000": { - "retcode": 0, - "retmsg": "Migrating model successfully. The Config of model has been modified automatically. New model id is: arbiter-100#guest-99#host-100#model, model version is: 202111091127392613050. Model files can be found at '/root/Codes/FATE-Flow/temp/fate_flow/arbiter#100#arbiter-100#guest-99#host-100#model_202111091127392613050.zip'." - } - }, - "guest": { - "9999": { - "retcode": 0, - "retmsg": "Migrating model successfully. The Config of model has been modified automatically. New model id is: arbiter-100#guest-99#host-100#model, model version is: 202111091127392613050. Model files can be found at '/root/Codes/FATE-Flow/temp/fate_flow/guest#99#arbiter-100#guest-99#host-100#model_202111091127392613050.zip'." - } - }, - "host": { - "10000": { - "retcode": 0, - "retmsg": "Migrating model successfully. The Config of model has been modified automatically. New model id is: arbiter-100#guest-99#host-100#model, model version is: 202111091127392613050. Model files can be found at '/root/Codes/FATE-Flow/temp/fate_flow/host#100#arbiter-100#guest-99#host-100#model_202111091127392613050.zip'." - } - } - }, - "guest": { - "9999": 0 - }, - "host": { - "10000": 0 - } - }, - "jobId": "202111091127392613050", - "retcode": 0, - "retmsg": "success" -} -``` - -### tag-list - -List tags of the model. - -``` bash -flow model tag-list -j -``` - -**Options** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| --------- | ---------- | ---------- | -------- | ----------- | -| job_id | `-j` | `--job_id` | No | Job ID | - -### tag-model - -Add or remove a tag from the model. - -```bash -flow model tag-model -j -t -flow model tag-model -j -t --remove -``` - -**Options** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| -------- | ------ | ------------ | -------- | -------------- | -| job_id | `-j` | `--job_id` | No | Job ID | -| tag_name | `-t` | `--tag-name` | No | Tag name | -| remove | | `--remove` | Yes | Remove the tag | - -### deploy - -Configure predict DSL. - -```bash -flow model deploy --model-id --model-version -``` - -**Options** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| -------------- | ---------- | ------------------ | -------- | ------------------------------------------------------------ | -| model_id | | `--model-id` | No | Model ID | -| model_version | | `--model-version` | No | Model version | -| cpn_list | | `--cpn-list` | Yes | Components list | -| cpn_path | | `--cpn-path` | Yes | Load components list from a file | -| dsl_path | | `--dsl-path` | Yes | Predict DSL file path | -| cpn_step_index | | `--cpn-step-index` | Yes | Specify a checkpoint model to replace the pipeline model
Use `:` to separate component name and step index
E.g. `--cpn-step-index cpn_a:123` | -| cpn_step_name | | `--cpn-step-name` | Yes | Specify a checkpoint model to replace the pipeline model.
Use `:` to separate component name and step name
E.g. `--cpn-step-name cpn_b:foobar` | - -**Example** - -```json -{ - "retcode": 0, - "retmsg": "success", - "data": { - "model_id": "arbiter-9999#guest-10000#host-9999#model", - "model_version": "202111032227378766180", - "arbiter": { - "party_id": 9999 - }, - "guest": { - "party_id": 10000 - }, - "host": { - "party_id": 9999 - }, - "detail": { - "arbiter": { - "party_id": { - "retcode": 0, - "retmsg": "deploy model of role arbiter 9999 success" - } - }, - "guest": { - "party_id": { - "retcode": 0, - "retmsg": "deploy model of role guest 10000 success" - } - }, - "host": { - "party_id": { - "retcode": 0, - "retmsg": "deploy model of role host 9999 success" - } - } - } - } -} -``` - -### get-predict-dsl - -Get predict DSL of the model. - -```bash -flow model get-predict-dsl --model-id --model-version -o ./examples/ -``` - -**Options** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| ------------- | ---------- | ----------------- | -------- | ------------- | -| model_id | | `--model-id` | No | Model ID | -| model_version | | `--model-version` | No | Model version | -| output_path | `-o` | `--output-path` | No | Output path | - -### get-predict-conf - -Get the template of predict config. - -```bash -flow model get-predict-conf --model-id --model-version -o ./examples/ -``` - -**Options** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| ------------- | ---------- | ----------------- | -------- | ------------- | -| model_id | | `--model-id` | No | Model ID | -| model_version | | `--model-version` | No | Model version | -| output_path | `-o` | `--output-path` | No | Output path | - -### get-model-info - -Get model information. - -```bash -flow model get-model-info --model-id --model-version -flow model get-model-info --model-id --model-version --detail -``` - -**Options** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| ------------- | ---------- | ----------------- | -------- | ---------------------------- | -| model_id | | `--model-id` | No | Model ID | -| model_version | | `--model-version` | No | Model version | -| role | `-r` | `--role` | Yes | Party role | -| party_id | `-p` | `--party-id` | Yes | Party ID | -| detail | | `--detail` | Yes | Display detailed information | - -### homo-convert - -Convert trained homogenous model to the format of another ML framework. - -```bash -flow model homo-convert -c examples/model/homo_convert_model.json -``` - -**Options** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| --------- | ---------- | ------------- | -------- | ---------------- | -| conf_path | `-c` | `--conf-path` | No | Config file path | - -### homo-deploy - -Deploy trained homogenous model to a target online serving system. Currently the supported target serving system is KFServing. - -```bash -flow model homo-deploy -c examples/model/homo_deploy_model.json -``` - -**Options** - -| Parameter | Short Flag | Long Flag | Optional | Description | -| --------- | ---------- | ------------- | -------- | ---------------- | -| conf_path | `-c` | `--conf-path` | No | Config file path | diff --git a/doc/cli/model.zh.md b/doc/cli/model.zh.md deleted file mode 100644 index db035d1df..000000000 --- a/doc/cli/model.zh.md +++ /dev/null @@ -1,375 +0,0 @@ -## Model - -### load - -向 Fate-Serving 加载 `deploy` 生成的模型。 - -```bash -flow model load -c examples/model/publish_load_model.json -flow model load -j -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| --------- | ------ | ------------- | -------- | -------- | -| conf_path | `-c` | `--conf-path` | 是 | 配置文件 | -| job_id | `-j` | `--job-id` | 是 | 任务 ID | - -**样例** - -```json -{ - "data": { - "detail": { - "guest": { - "9999": { - "retcode": 0, - "retmsg": "success" - } - }, - "host": { - "10000": { - "retcode": 0, - "retmsg": "success" - } - } - }, - "guest": { - "9999": 0 - }, - "host": { - "10000": 0 - } - }, - "jobId": "202111091122168817080", - "retcode": 0, - "retmsg": "success" -} -``` - -### bind - -向 Fate-Serving 绑定 `deploy` 生成的模型。 - -```bash -flow model bind -c examples/model/bind_model_service.json -flow model bind -c examples/model/bind_model_service.json -j -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| --------- | ------ | ------------- | -------- | -------- | -| conf_path | `-c` | `--conf-path` | 否 | 配置文件 | -| job_id | `-j` | `--job-id` | 是 | 任务 ID | - -**样例** - -```json -{ - "retcode": 0, - "retmsg": "service id is 123" -} -``` - -### import - -从本地或存储引擎中导入模型。 - -```bash -flow model import -c examples/model/import_model.json -flow model import -c examples/model/restore_model.json --from-database -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| ------------- | ------ | ----------------- | -------- | -------------------- | -| conf_path | `-c` | `--conf-path` | 否 | 配置文件 | -| from_database | | `--from-database` | 是 | 从存储引擎中导入模型 | - -**样例** - -```json -{ - "data": { - "job_id": "202208261102212849780", - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "foobar", - "party_id": "9999", - "role": "guest" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### export - -导出模型到本地或存储引擎中。 - -```bash -flow model export -c examples/model/export_model.json -flow model export -c examples/model/store_model.json --to-database -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| ----------- | ------ | --------------- | -------- | ---------------------- | -| conf_path | `-c` | `--conf-path` | 否 | 配置文件 | -| to_database | | `--to-database` | 是 | 将模型导出到存储引擎中 | - -**样例** - -```json -{ - "data": { - "board_url": "http://127.0.0.1:8080/index.html#/dashboard?job_id=202111091124582110490&role=local&party_id=0", - "code": 0, - "dsl_path": "/root/Codes/FATE-Flow/jobs/202111091124582110490/job_dsl.json", - "job_id": "202111091124582110490", - "logs_directory": "/root/Codes/FATE-Flow/logs/202111091124582110490", - "message": "success", - "model_info": { - "model_id": "local-0#model", - "model_version": "202111091124582110490" - }, - "pipeline_dsl_path": "/root/Codes/FATE-Flow/jobs/202111091124582110490/pipeline_dsl.json", - "runtime_conf_on_party_path": "/root/Codes/FATE-Flow/jobs/202111091124582110490/local/0/job_runtime_on_party_conf.json", - "runtime_conf_path": "/root/Codes/FATE-Flow/jobs/202111091124582110490/job_runtime_conf.json", - "train_runtime_conf_path": "/root/Codes/FATE-Flow/jobs/202111091124582110490/train_runtime_conf.json" - }, - "jobId": "202111091124582110490", - "retcode": 0, - "retmsg": "success" -} -``` - -### migrate - -迁移模型。 - -```bash -flow model migrate -c examples/model/migrate_model.json -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| --------- | ------ | ------------- | -------- | -------- | -| conf_path | `-c` | `--conf-path` | 否 | 配置文件 | - -**样例** - -```json -{ - "data": { - "arbiter": { - "10000": 0 - }, - "detail": { - "arbiter": { - "10000": { - "retcode": 0, - "retmsg": "Migrating model successfully. The configuration of model has been modified automatically. New model id is: arbiter-100#guest-99#host-100#model, model version is: 202111091127392613050. Model files can be found at '/root/Codes/FATE-Flow/temp/fate_flow/arbiter#100#arbiter-100#guest-99#host-100#model_202111091127392613050.zip'." - } - }, - "guest": { - "9999": { - "retcode": 0, - "retmsg": "Migrating model successfully. The configuration of model has been modified automatically. New model id is: arbiter-100#guest-99#host-100#model, model version is: 202111091127392613050. Model files can be found at '/root/Codes/FATE-Flow/temp/fate_flow/guest#99#arbiter-100#guest-99#host-100#model_202111091127392613050.zip'." - } - }, - "host": { - "10000": { - "retcode": 0, - "retmsg": "Migrating model successfully. The configuration of model has been modified automatically. New model id is: arbiter-100#guest-99#host-100#model, model version is: 202111091127392613050. Model files can be found at '/root/Codes/FATE-Flow/temp/fate_flow/host#100#arbiter-100#guest-99#host-100#model_202111091127392613050.zip'." - } - } - }, - "guest": { - "9999": 0 - }, - "host": { - "10000": 0 - } - }, - "jobId": "202111091127392613050", - "retcode": 0, - "retmsg": "success" -} -``` - -### tag-list - -获取模型的标签列表。 - -``` bash -flow model tag-list -j -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| ------ | ------ | ---------- | -------- | ------- | -| job_id | `-j` | `--job_id` | 否 | 任务 ID | - -### tag-model - -从模型中添加或删除标签。 - -```bash -flow model tag-model -j -t -flow model tag-model -j -t --remove -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| -------- | ------ | ------------ | -------- | -------------- | -| job_id | `-j` | `--job_id` | 否 | 任务 ID | -| tag_name | `-t` | `--tag-name` | 否 | 标签名 | -| remove | | `--remove` | 是 | 移除指定的标签 | - -### deploy - -配置预测 DSL。 - -```bash -flow model deploy --model-id --model-version -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| -------------- | ------ | ------------------ | -------- | ------------------------------------------------------------ | -| model_id | | `--model-id` | 否 | 模型 ID | -| model_version | | `--model-version` | 否 | 模型版本 | -| cpn_list | | `--cpn-list` | 是 | 组件列表 | -| cpn_path | | `--cpn-path` | 是 | 从文件中读入组件列表 | -| dsl_path | | `--dsl-path` | 是 | 预测 DSL 文件 | -| cpn_step_index | | `--cpn-step-index` | 是 | 用指定的 Checkpoint 模型替换 Pipeline 模型
使用 `:` 分隔 component name 与 step index
例如 `--cpn-step-index cpn_a:123` | -| cpn_step_name | | `--cpn-step-name` | 是 | 用指定的 Checkpoint 模型替换 Pipeline 模型
使用 `:` 分隔 component name 与 step name
例如 `--cpn-step-name cpn_b:foobar` | - -**样例** - -```json -{ - "retcode": 0, - "retmsg": "success", - "data": { - "model_id": "arbiter-9999#guest-10000#host-9999#model", - "model_version": "202111032227378766180", - "arbiter": { - "party_id": 9999 - }, - "guest": { - "party_id": 10000 - }, - "host": { - "party_id": 9999 - }, - "detail": { - "arbiter": { - "party_id": { - "retcode": 0, - "retmsg": "deploy model of role arbiter 9999 success" - } - }, - "guest": { - "party_id": { - "retcode": 0, - "retmsg": "deploy model of role guest 10000 success" - } - }, - "host": { - "party_id": { - "retcode": 0, - "retmsg": "deploy model of role host 9999 success" - } - } - } - } -} -``` - -### get-predict-dsl - -获取预测 DSL。 - -```bash -flow model get-predict-dsl --model-id --model-version -o ./examples/ -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| ------------- | ------ | ----------------- | -------- | -------- | -| model_id | | `--model-id` | 否 | 模型 ID | -| model_version | | `--model-version` | 否 | 模型版本 | -| output_path | `-o` | `--output-path` | 否 | 输出路径 | - -### get-predict-conf - -获取模型预测模板。 - -```bash -flow model get-predict-conf --model-id --model-version -o ./examples/ -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| ------------- | ------ | ----------------- | -------- | -------- | -| model_id | | `--model-id` | 否 | 模型 ID | -| model_version | | `--model-version` | 否 | 模型版本 | -| output_path | `-o` | `--output-path` | 否 | 输出路径 | - -### get-model-info - -获取模型信息。 - -```bash -flow model get-model-info --model-id --model-version -flow model get-model-info --model-id --model-version --detail -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| ------------- | ------ | ----------------- | -------- | ------------ | -| model_id | | `--model-id` | 否 | 模型 ID | -| model_version | | `--model-version` | 否 | 模型版本 | -| role | `-r` | `--role` | 是 | Party 角色 | -| party_id | `-p` | `--party-id` | 是 | Party ID | -| detail | | `--detail` | 是 | 展示详细信息 | - -### homo-convert - -基于横向训练的模型，生成其他 ML 框架的模型文件。 - -```bash -flow model homo-convert -c examples/model/homo_convert_model.json -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| --------- | ------ | ------------- | -------- | -------- | -| conf_path | `-c` | `--conf-path` | 否 | 配置文件 | - -### homo-deploy - -将横向训练后使用 `homo-convert` 生成的模型部署到在线推理系统中，当前支持创建基于 KFServing 的推理服务。 - -```bash -flow model homo-deploy -c examples/model/homo_deploy_model.json -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 可选参数 | 说明 | -| --------- | ------ | ------------- | -------- | -------- | -| conf_path | `-c` | `--conf-path` | 否 | 配置文件 | diff --git a/doc/cli/privilege.md b/doc/cli/privilege.md deleted file mode 100644 index 1da7fea63..000000000 --- a/doc/cli/privilege.md +++ /dev/null @@ -1,150 +0,0 @@ -## Privilege - -### grant - -Add privileges - -```bash -flow privilege grant -c fateflow/examples/permission/grant.json -``` - -**Options** - -| parameter name | required | type | description | -|:----------|:----|:-------|-------------------------------------------------------------------------------------| -| party_id | yes | string | site id | -| component | no | string | component name, can be split by "," for multiple components, "*" for all components | -| dataset | no | object | list of datasets | - - -**sample** -```json -{ - "party_id": 10000, - "component": "reader,dataio", - "dataset": [ - { - "namespace": "experiment", - "name": "breast_hetero_guest" - }, - { - "namespace": "experiment", - "name": "breast_hetero_host" - } - ] -} -``` - -**return** - -| parameter name | type | description | -| ------- | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | - -**Sample** - -```shell -{ - "retcode": 0, - "retmsg": "success" -} -``` - -### delete - -Delete permissions - -```bash -flow privilege delete -c fateflow/examples/permission/delete.json -``` -**Options** - -| parameter name | required | type | description | -|:----------|:----|:-------|--------------------------| -| party_id | yes | string | site_id | -| component | no | string | component name, can be split by "," for multiple components, "*" for all components | -| dataset | no | object | list of datasets, "*" is all datasets | - -**sample** -```json -{ - "party_id": 10000, - "component": "reader,dataio", - "dataset": [ - { - "namespace": "experiment", - "name": "breast_hetero_guest" - }, - { - "namespace": "experiment", - "name": "breast_hetero_host" - } - ] -} -``` - -**return** - -| parameter name | type | description | -| ------- | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | - -**Sample** - -```shell -{ - "retcode": 0, - "retmsg": "success" -} -``` - -### query - -Query permissions - -```bash -flow privilege query -p 10000 -``` - -**Options** - -| parameters | short-format | long-format | required | type | description | -| :-------- |:-----|:-------------| :--- | :----- |------| -| party_id | `-p` | `--party-id` | yes | string | site id | - -**returns** - - -| parameter name | type | description | -| ------- | :----- | -------- | -| retcode | int | return-code | -| retmsg | string | Return information | -| data | object | return data | - -**Sample** - -```json -{ - "data": { - "component": [ - "reader", - "dataio" - ], - "dataset": [ - { - "name": "breast_hetero_guest", - "namespace": "experiment" - }, - { - "name": "breast_hetero_host", - "namespace": "experiment" - } - ] - }, - "retcode": 0, - "retmsg": "success" -} - -``` \ No newline at end of file diff --git a/doc/cli/privilege.zh.md b/doc/cli/privilege.zh.md deleted file mode 100644 index c9944b6de..000000000 --- a/doc/cli/privilege.zh.md +++ /dev/null @@ -1,163 +0,0 @@ -## Privilege - -### grant - -添加权限 - -```bash -flow privilege grant -c fateflow/examples/permission/grant.json -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 必选 | 类型 | 说明 | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| conf_path | `-c` |`--conf-path` |是 | string | 配置路径 | - -注: conf_path为参数路径，具体参数如下 - -| 参数名 | 必选 | 类型 | 说明 | -|:----------|:----|:-------|----------| -| party_id | 是 | string | 站点id | -| component | 否 | string | 组件名，可用","分割多个组件，"*"为所有组件 | -| dataset | 否 | object | 数据集列表 | - - -**样例** -```json -{ - "party_id": 10000, - "component": "reader,dataio", - "dataset": [ - { - "namespace": "experiment", - "name": "breast_hetero_guest" - }, - { - "namespace": "experiment", - "name": "breast_hetero_host" - } - ] -} -``` - -**返回** - -| 参数名 | 类型 | 说明 | -| ------- | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | - -**样例** - -```shell -{ - "retcode": 0, - "retmsg": "success" -} -``` - -### delete - -删除权限 - -```bash -flow privilege delete -c fateflow/examples/permission/delete.json -``` -**选项** - -| 参数 | 短格式 | 长格式 | 必选 | 类型 | 说明 | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| conf_path | `-c` |`--conf-path` |是 | string | 配置路径 | - - -注: conf_path为参数路径，具体参数如下 - -| 参数名 | 必选 | 类型 | 说明 | -|:----------|:----|:-------|--------------------------| -| party_id | 是 | string | 站点id | -| component | 否 | string | 组件名，可用","分割多个组件，"*"为所有组件 | -| dataset | 否 | object | 数据集列表， "*"为所有数据集 | - -**样例** -```json -{ - "party_id": 10000, - "component": "reader,dataio", - "dataset": [ - { - "namespace": "experiment", - "name": "breast_hetero_guest" - }, - { - "namespace": "experiment", - "name": "breast_hetero_host" - } - ] -} -``` - -**返回** - -| 参数名 | 类型 | 说明 | -| ------- | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | - -**样例** - -```shell -{ - "retcode": 0, - "retmsg": "success" -} -``` - -### query - -查询权限 - -```bash -flow privilege query -p 10000 -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 必选 | 类型 | 说明 | -| :-------- |:-----|:-------------| :--- | :----- |------| -| party_id | `-p` | `--party-id` |是 | string | 站点id | - -**返回** - - -| 参数名 | 类型 | 说明 | -| ------- | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -**样例** - -```json -{ - "data": { - "component": [ - "reader", - "dataio" - ], - "dataset": [ - { - "name": "breast_hetero_guest", - "namespace": "experiment" - }, - { - "name": "breast_hetero_host", - "namespace": "experiment" - } - ] - }, - "retcode": 0, - "retmsg": "success" -} - -``` diff --git a/doc/cli/provider.md b/doc/cli/provider.md deleted file mode 100644 index 453343d9a..000000000 --- a/doc/cli/provider.md +++ /dev/null @@ -1,179 +0,0 @@ -## Provider - -### list - -List all current component providers and information about the components they provide - -```bash -flow provider list [options] -``` - -**Options** - -**Returns** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | dict | 返回数据 | - -**Example** - -output: - -```json -{ - "data": { - "fate": { - "1.9.0": { - "class_path": { - "anonymous_generator": "util.anonymous_generator_util.Anonymous", - "data_format": "util.data_format_preprocess.DataFormatPreProcess", - "feature_instance": "feature.instance.Instance", - "feature_vector": "feature.sparse_vector.SparseVector", - "hetero_model_merge": "protobuf.model_merge.merge_hetero_models.hetero_model_merge", - "homo_model_convert": "protobuf.homo_model_convert.homo_model_convert", - "interface": "components.components.Components", - "model": "protobuf.generated", - "model_migrate": "protobuf.model_migrate.model_migrate" - }, - "components": [ - "heterodatasplit", - "psi", - "heterofastsecureboost", - "heterofeaturebinning", - "scorecard", - "sampleweight", - "homosecureboost", - "onehotencoder", - "secureinformationretrieval", - "homoonehotencoder", - "datatransform", - "dataio", - "heterosshelinr", - "intersection", - "homofeaturebinning", - "secureaddexample", - "union", - "datastatistics", - "columnexpand", - "homonn", - "labeltransform", - "heterosecureboost", - "heterofeatureselection", - "heterolr", - "feldmanverifiablesum", - "heteropoisson", - "evaluation", - "federatedsample", - "homodatasplit", - "ftl", - "localbaseline", - "featurescale", - "featureimputation", - "heteropearson", - "heterokmeans", - "heteronn", - "heterolinr", - "spdztest", - "heterosshelr", - "homolr" - ], - "path": "${FATE_PROJECT_BASE}/python/federatedml", - "python": "" - }, - "default": { - "version": "1.9.0" - } - }, - "fate_flow": { - "1.9.0": { - "class_path": { - "anonymous_generator": "util.anonymous_generator_util.Anonymous", - "data_format": "util.data_format_preprocess.DataFormatPreProcess", - "feature_instance": "feature.instance.Instance", - "feature_vector": "feature.sparse_vector.SparseVector", - "hetero_model_merge": "protobuf.model_merge.merge_hetero_models.hetero_model_merge", - "homo_model_convert": "protobuf.homo_model_convert.homo_model_convert", - "interface": "components.components.Components", - "model": "protobuf.generated", - "model_migrate": "protobuf.model_migrate.model_migrate" - }, - "components": [ - "writer", - "modelrestore", - "upload", - "apireader", - "modelstore", - "cacheloader", - "modelloader", - "download", - "reader" - ], - "path": "${FATE_FLOW_BASE}/python/fate_flow", - "python": "" - }, - "default": { - "version": "1.9.0" - } - } - }, - "retcode": 0, - "retmsg": "success" -} -``` - -Contains the `name`, `version number`, `codepath`, `list of provided components` - -### register - -Register a component provider - -```bash -flow provider register [options] -``` - -**Options** - -| 参数名 | 必选 | 类型 | 说明 | -| :--------------------- | :--- | :----- | ------------------------------| -| -c, --conf-path | 是 | string | 配置路径 | - -**Returns** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | - -**Example** - -```bash -flow provider register -c $FATE_FLOW_BASE/examples/other/register_provider.json -``` - -conf: - -```json -{ - "name": "fate", - "version": "1.7.1", - "path": "${FATE_FLOW_BASE}/python/component_plugins/fateb/python/federatedml" -} -``` - -output: - -```json -{ - "data": { - "flow-xxx-9380": { - "retcode": 0, - "retmsg": "success" - } - }, - "retcode": 0, - "retmsg": "success" -} -``` diff --git a/doc/cli/provider.zh.md b/doc/cli/provider.zh.md deleted file mode 100644 index 530ae2be7..000000000 --- a/doc/cli/provider.zh.md +++ /dev/null @@ -1,180 +0,0 @@ -## Provider - -### list - -列出当前所有组件提供者及其提供组件信息 - -```bash -flow provider list [options] -``` - -**选项** - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | dict | 返回数据 | - -**样例** - -输出: - -```json -{ - "data": { - "fate": { - "1.9.0": { - "class_path": { - "anonymous_generator": "util.anonymous_generator_util.Anonymous", - "data_format": "util.data_format_preprocess.DataFormatPreProcess", - "feature_instance": "feature.instance.Instance", - "feature_vector": "feature.sparse_vector.SparseVector", - "hetero_model_merge": "protobuf.model_merge.merge_hetero_models.hetero_model_merge", - "homo_model_convert": "protobuf.homo_model_convert.homo_model_convert", - "interface": "components.components.Components", - "model": "protobuf.generated", - "model_migrate": "protobuf.model_migrate.model_migrate" - }, - "components": [ - "heterodatasplit", - "psi", - "heterofastsecureboost", - "heterofeaturebinning", - "scorecard", - "sampleweight", - "homosecureboost", - "onehotencoder", - "secureinformationretrieval", - "homoonehotencoder", - "datatransform", - "dataio", - "heterosshelinr", - "intersection", - "homofeaturebinning", - "secureaddexample", - "union", - "datastatistics", - "columnexpand", - "homonn", - "labeltransform", - "heterosecureboost", - "heterofeatureselection", - "heterolr", - "feldmanverifiablesum", - "heteropoisson", - "evaluation", - "federatedsample", - "homodatasplit", - "ftl", - "localbaseline", - "featurescale", - "featureimputation", - "heteropearson", - "heterokmeans", - "heteronn", - "heterolinr", - "spdztest", - "heterosshelr", - "homolr" - ], - "path": "${FATE_PROJECT_BASE}/python/federatedml", - "python": "" - }, - "default": { - "version": "1.9.0" - } - }, - "fate_flow": { - "1.9.0": { - "class_path": { - "anonymous_generator": "util.anonymous_generator_util.Anonymous", - "data_format": "util.data_format_preprocess.DataFormatPreProcess", - "feature_instance": "feature.instance.Instance", - "feature_vector": "feature.sparse_vector.SparseVector", - "hetero_model_merge": "protobuf.model_merge.merge_hetero_models.hetero_model_merge", - "homo_model_convert": "protobuf.homo_model_convert.homo_model_convert", - "interface": "components.components.Components", - "model": "protobuf.generated", - "model_migrate": "protobuf.model_migrate.model_migrate" - }, - "components": [ - "writer", - "modelrestore", - "upload", - "apireader", - "modelstore", - "cacheloader", - "modelloader", - "download", - "reader" - ], - "path": "${FATE_FLOW_BASE}/python/fate_flow", - "python": "" - }, - "default": { - "version": "1.9.0" - } - } - }, - "retcode": 0, - "retmsg": "success" -} -``` - -包含`组件提供者`的`名称`, `版本号`, `代码路径`, `提供的组件列表` - -### register - -注册一个组件提供者 - -```bash -flow provider register [options] -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :--------------------- | :--- | :----- | ------------------------------| -| -c, --conf-path | 是 | string | 配置路径 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | - -**样例** - -```bash -flow provider register -c $FATE_FLOW_BASE/examples/other/register_provider.json -``` - -配置文件： - -```json -{ - "name": "fate", - "version": "1.7.1", - "path": "${FATE_FLOW_BASE}/python/component_plugins/fateb/python/federatedml" -} -``` - -输出: - -```json -{ - "data": { - "flow-xxx-9380": { - "retcode": 0, - "retmsg": "success" - } - }, - "retcode": 0, - "retmsg": "success" -} - -``` diff --git a/doc/cli/resource.md b/doc/cli/resource.md deleted file mode 100644 index 9576e6fa0..000000000 --- a/doc/cli/resource.md +++ /dev/null @@ -1,89 +0,0 @@ -## Resource - -### query - -For querying fate system resources - -```bash -flow resource query -``` - -**Options** - -**Returns** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | -| data | object | return data | - -**Example** - -```json -{ - "data": { - "computing_engine_resource": { - "f_cores": 32, - "f_create_date": "2021-09-21 19:32:59", - "f_create_time": 1632223979564, - "f_engine_config": { - "cores_per_node": 32, - "nodes": 1 - }, - "f_engine_entrance": "fate_on_eggroll", - "f_engine_name": "EGGROLL", - "f_engine_type": "computing", - "f_memory": 0, - "f_nodes": 1, - "f_remaining_cores": 32, - "f_remaining_memory": 0, - "f_update_date": "2021-11-08 16:56:38", - "f_update_time": 1636361798812 - }, - "use_resource_job": [] - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### return - -Resources for returning a job - -```bash -flow resource return [options] -``` - -**Options** - -| parameter name | required | type | description | -| :----- | :--- | :----- | ------ | -| job_id | yes | string | job_id | - -**Returns** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | -| data | object | return data | - -**Example** - -```json -{ - "data": [ - { - "job_id": "202111081612427726750", - "party_id": "8888", - "resource_in_use": true, - "resource_return_status": true, - "role": "guest" - } - ], - "retcode": 0, - "retmsg": "success" -} -``` diff --git a/doc/cli/resource.zh.md b/doc/cli/resource.zh.md deleted file mode 100644 index 93b01aa31..000000000 --- a/doc/cli/resource.zh.md +++ /dev/null @@ -1,89 +0,0 @@ -## Resource - -### query - -用于查询fate系统资源 - -```bash -flow resource query -``` - -**选项** - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -**样例** - -```json -{ - "data": { - "computing_engine_resource": { - "f_cores": 32, - "f_create_date": "2021-09-21 19:32:59", - "f_create_time": 1632223979564, - "f_engine_config": { - "cores_per_node": 32, - "nodes": 1 - }, - "f_engine_entrance": "fate_on_eggroll", - "f_engine_name": "EGGROLL", - "f_engine_type": "computing", - "f_memory": 0, - "f_nodes": 1, - "f_remaining_cores": 32, - "f_remaining_memory": 0, - "f_update_date": "2021-11-08 16:56:38", - "f_update_time": 1636361798812 - }, - "use_resource_job": [] - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### return - -用于归还某个job的资源 - -```bash -flow resource return [options] -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :----- | :--- | :----- | ------ | -| job_id | 是 | string | 任务id | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -**样例** - -```json -{ - "data": [ - { - "job_id": "202111081612427726750", - "party_id": "8888", - "resource_in_use": true, - "resource_return_status": true, - "role": "guest" - } - ], - "retcode": 0, - "retmsg": "success" -} -``` diff --git a/doc/cli/server.md b/doc/cli/server.md deleted file mode 100644 index 92f1e9a7d..000000000 --- a/doc/cli/server.md +++ /dev/null @@ -1,111 +0,0 @@ -## Server - -### versions - -List all relevant system version numbers - -```bash -flow server versions -``` - -**Options** - -None - -**Returns** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | -| data | dict | return data | -| jobId | string | job id | - -**Example** - -```bash -flow server versions -``` - -Output: - -```json -{ - "data": { - "API": "v1", - "CENTOS": "7.2", - "EGGROLL": "2.4.0", - "FATE": "1.7.0", - "FATEBoard": "1.7.0", - "FATEFlow": "1.7.0", - "JDK": "8", - "MAVEN": "3.6.3", - "PYTHON": "3.6.5", - "SPARK": "2.4.1", - "UBUNTU": "16.04" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### reload - -The following configuration items will take effect again after `reload` - - - All configurations after # engine services in $FATE_PROJECT_BASE/conf/service_conf.yaml - - All configurations in $FATE_FLOW_BASE/python/fate_flow/job_default_config.yaml - -```bash -flow server reload -``` - -**Options** - -None - -**Returns** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | -| data | dict | return data | -| jobId | string | job id | - -**Example** - -```bash -flow server reload -``` - -Output: - -```json -{ - "data": { - "job_default_config": { - "auto_retries": 0, - "auto_retry_delay": 1, - "default_component_provider_path": "component_plugins/fate/python/federatedml", - "end_status_job_scheduling_time_limit": 300000, - "end_status_job_scheduling_updates": 1, - "federated_command_trys": 3, - "federated_status_collect_type": "PUSH", - "job_timeout": 259200, - "max_cores_percent_per_job": 1, - "output_data_summary_count_limit": 100, - "remote_request_timeout": 30000, - "task_cores": 4, - "task_memory": 0, - "task_parallelism": 1, - "total_cores_overweight_percent": 1, - "total_memory_overweight_percent": 1, - "upload_max_bytes": 4194304000 - }, - "service_registry": null - }, - "retcode": 0, - "retmsg": "success" -} -``` diff --git a/doc/cli/server.zh.md b/doc/cli/server.zh.md deleted file mode 100644 index 44c65e7b4..000000000 --- a/doc/cli/server.zh.md +++ /dev/null @@ -1,111 +0,0 @@ -## Server - -### versions - -列出所有相关系统版本号 - -```bash -flow server versions -``` - -**选项** - -无 - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | dict | 返回数据 | -| jobId | string | 作业id | - -**样例** - -```bash -flow server versions -``` - -输出: - -```json -{ - "data": { - "API": "v1", - "CENTOS": "7.2", - "EGGROLL": "2.4.0", - "FATE": "1.7.0", - "FATEBoard": "1.7.0", - "FATEFlow": "1.7.0", - "JDK": "8", - "MAVEN": "3.6.3", - "PYTHON": "3.6.5", - "SPARK": "2.4.1", - "UBUNTU": "16.04" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### reload - -如下配置项在`reload`后会重新生效 - - - $FATE_PROJECT_BASE/conf/service_conf.yaml中# engine services后的所有配置 - - $FATE_FLOW_BASE/python/fate_flow/job_default_config.yaml中所有配置 - -```bash -flow server reload -``` - -**选项** - -无 - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | dict | 返回数据 | -| jobId | string | 作业id | - -**样例** - -```bash -flow server reload -``` - -输出: - -```json -{ - "data": { - "job_default_config": { - "auto_retries": 0, - "auto_retry_delay": 1, - "default_component_provider_path": "component_plugins/fate/python/federatedml", - "end_status_job_scheduling_time_limit": 300000, - "end_status_job_scheduling_updates": 1, - "federated_command_trys": 3, - "federated_status_collect_type": "PUSH", - "job_timeout": 259200, - "max_cores_percent_per_job": 1, - "output_data_summary_count_limit": 100, - "remote_request_timeout": 30000, - "task_cores": 4, - "task_memory": 0, - "task_parallelism": 1, - "total_cores_overweight_percent": 1, - "total_memory_overweight_percent": 1, - "upload_max_bytes": 4194304000 - }, - "service_registry": null - }, - "retcode": 0, - "retmsg": "success" -} -``` diff --git a/doc/cli/table.md b/doc/cli/table.md deleted file mode 100644 index baf0ca233..000000000 --- a/doc/cli/table.md +++ /dev/null @@ -1,320 +0,0 @@ -## Table - -### info - -Query information about the fate table (real storage address, number, schema, etc.) - -```bash -flow table info [options] -``` - -**options** - -| parameters | short-format | long-format | required | type | description | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| table_name | `-t` |`--table-name` |yes | string | fate table name | -| namespace | `-n` |`--namespace` | yes |string | fate table namespace | - -**returns** -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return information | -| data | object | return data | - -Sample - -```json -{ - "data": { - "address": { - "home": null, - "name": "breast_hetero_guest", - "namespace": "experiment" - }, - "count": 569, - "exists": 1, - "namespace": "experiment", - "partition": 4, - "schema": { - "header": "y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9", - "sid": "id" - }, - "table_name": "breast_hetero_guest" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### delete - -You can delete table data with table delete - -```bash -flow table delete [options] -``` - -**Options** - -| parameters | short-format | long-format | required | type | description | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| table_name | `-t` |`--table-name` |yes | string | fate table name | -| namespace | `-n` |`--namespace` | yes |string | fate table namespace | - -**returns** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | -| data | object | return data | - -Sample - -```json -{ - "data": { - "namespace": "xxx", - "table_name": "xxx" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### bind - -Real storage addresses can be mapped to fate storage tables via table bind - -```bash -flow table bind [options] -``` - -**options** - -| parameters | short format | long format | required | type | description | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| conf_path | `-c` | `--conf-path` | yes | string | configuration-path | - -Note: conf_path is the parameter path, the specific parameters are as follows - -| parameter_name | required | type | description | -| :------------- | :--- | :----- | ------------------------------------- | -| name | yes | string | fate table name | -| namespace | yes | string | fate table namespace | -| engine | yes | string | storage engine, supports "HDFS", "MYSQL", "PATH" | -| yes | object | real storage address | -| drop | no | int | Overwrite previous information | -| head | no | int | Whether there is a data table header | -| id_delimiter | no | string | Data separator | -| id_column | no | string | id field | -| feature_column | no | array | feature_field | - -**mete information** - -| parameter name | required | type | description | -|:---------------------|:----|:-------|-------------------------------------------| -| input_format | no | string | The format of the data (danse, svmlight, tag:value), used to determine | -| delimiter | no | string | The data separator, default "," | -| tag_with_value | no | bool | Valid for tag data format, whether to carry value | -| tag_value_delimiter | no | string | tag:value data separator, default ":" | -| with_match_id | no | bool | Whether or not to carry match id | -| with_match_id | no | object | The name of the id column, effective when extend_sid is enabled, e.g., ["email", "phone"] | -| id_range | no | object | For tag/svmlight format data, which columns are ids | -| exclusive_data_type | no | string | The format of the special type data columns | -| data_type | no | string | Column data type, default "float64 | -| with_label | no | bool | Whether to have a label, default False | -| label_name | no | string | The name of the label, default "y" | -| label_type | no | string | Label type, default "int" | - -**In version 1.9.0 and later, if the meta parameter is passed in during the table bind phase, no anonymous information about the feature is generated directly. -The feature anonymization information of the original data will be updated after the data has passed through the reader component once** - -**Sample** - -- hdfs - -```json -{ - "namespace": "experiment", - "name": "breast_hetero_guest", - "engine": "HDFS", - "address": { - "name_node": "hdfs://fate-cluster", - "path": "/data/breast_hetero_guest.csv" - }, - "id_delimiter": ",", - "head": 1, - "partitions": 10 -} -``` - -- mysql - -```json -{ - "engine": "MYSQL", - "address": { - "user": "fate", - "passwd": "fate", - "host": "127.0.0.1", - "port": 3306, - "db": "experiment", - "name": "breast_hetero_guest" - }, - "namespace": "experiment", - "name": "breast_hetero_guest", - "head": 1, - "id_delimiter": ",", - "partitions": 10, - "id_column": "id", - "feature_column": "y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9" -} -``` - -- PATH - -```json -{ - "namespace": "xxx", - "name": "xxx", - "engine": "PATH", - "address": { - "path": "xxx" - } -} -``` -**return** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return information | -| data | object | return data | - -Sample - -```json -{ - "data": { - "namespace": "xxx", - "table_name": "xxx" - }, - "retcode": 0, - "retmsg": "success" -} -``` - - -### disable - -Tables can be made unavailable by table disable - -```bash -flow table disable [options] -``` - -**Options** - -| parameters | short-format | long-format | required | type | description | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| table_name | `-t` | `--table-name` | yes | string | fate table name | -| namespace | `-n` |`--namespace` | yes |string | fate table namespace | - -**returns** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return information | -| data | object | return data | - -Sample - -```json -{ - "data": { - "namespace": "xxx", - "table_name": "xxx" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### enable - -Tables can be made available with table enable - -```bash -flow table enable [options] -``` - -**Options** - -| parameters | short-format | long-format | required | type | description | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| table_name | `-t` | `--table-name` | yes | string | fate table name | -| namespace | `-n` |`--namespace` | yes |string | fate table namespace | - - -**returns** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return information | -| data | object | return data | - -Sample - -```json -{ - "data": [{ - "namespace": "xxx", - "table_name": "xxx" - }], - "retcode": 0, - "retmsg": "success" -} -``` - -### disable-delete - -Tables that are currently unavailable can be deleted with disable-delete - -```bash -flow table disable-delete -``` - - -**return** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return-code | -| retmsg | string | return information | -| data | object | return data | - -Sample - -```json -{ - "data": [ - { - "namespace": "xxx", - "table_name": "xxx" - }, - { - "namespace": "xxx", - "table_name": "xxx" - } - ], - "retcode": 0, - "retmsg": "success" -} -``` - - - \ No newline at end of file diff --git a/doc/cli/table.zh.md b/doc/cli/table.zh.md deleted file mode 100644 index fcb579fb7..000000000 --- a/doc/cli/table.zh.md +++ /dev/null @@ -1,318 +0,0 @@ -## Table - -### info - -用于查询fate表的相关信息(真实存储地址,数量,schema等) - -```bash -flow table info [options] -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 必选 | 类型 | 说明 | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| table_name | `-t` |`--table-name` |是 | string | fate表名 | -| namespace | `-n` |`--namespace` | 是 |string | fate表命名空间 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -样例 - -```json -{ - "data": { - "address": { - "home": null, - "name": "breast_hetero_guest", - "namespace": "experiment" - }, - "count": 569, - "exist": 1, - "namespace": "experiment", - "partition": 4, - "schema": { - "header": "y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9", - "sid": "id" - }, - "table_name": "breast_hetero_guest" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### delete - -可通过table delete删除表数据 - -```bash -flow table delete [options] -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 必选 | 类型 | 说明 | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| table_name | `-t` |`--table-name` |是 | string | fate表名 | -| namespace | `-n` |`--namespace` | 是 |string | fate表命名空间 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -样例 - -```json -{ - "data": { - "namespace": "xxx", - "table_name": "xxx" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### bind - -可通过table bind将真实存储地址映射到fate存储表 - -```bash -flow table bind [options] -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 必选 | 类型 | 说明 | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| conf_path | `-c` |`--conf-path` |是 | string | 配置路径 | - -注: conf_path为参数路径，具体参数如下 - -| 参数名 | 必选 | 类型 | 说明 | -| :------------- | :--- | :----- | ------------------------------------- | -| name | 是 | string | fate表名 | -| namespace | 是 | string | fate表命名空间 | -| engine | 是 | string | 存储引擎, 支持"HDFS", "MYSQL", "PATH" | -| adress | 是 | object | 真实存储地址 | -| drop | 否 | int | 覆盖以前的信息 | -| head | 否 | int | 是否有数据表头 | -| id_delimiter | 否 | string | 数据分隔符 | -| id_column | 否 | string | id字段 | -| feature_column | 否 | array | 特征字段 | - -**mete信息** - -| 参数名 | 必选 | 类型 | 说明 | -|:---------------------|:----|:-------|-------------------------------------------| -| input_format | 否 | string | 数据格式(danse、svmlight、tag:value)，用来判断 | -| delimiter | 否 | string | 数据分隔符,默认"," | -| tag_with_value | 否 | bool | 对tag的数据格式生效，是否携带value | -| tag_value_delimiter | 否 | string | tag:value数据分隔符，默认":" | -| with_match_id | 否 | bool | 是否携带match id | -| id_list | 否 | object | id列名称，开启extend_sid下生效，如：["imei", "phone"] | -| id_range | 否 | object | 对于tag/svmlight格式数据，哪几列为id | -| exclusive_data_type | 否 | string | 特殊类型数据列格式 | -| data_type | 否 | string | 列数据类型，默认"float64 | -| with_label | 否 | bool | 是否有标签，默认False | -| label_name | 否 | string | 标签名，默认"y" | -| label_type | 否 | string | 标签类型, 默认"int" | - -**注：在1.9.0及之后的版本中，若在table bind阶段传入meta参数，并不会直接生成特征的匿名信息。 -当数据经过了一次reader组件后会更新原始数据的特征匿名信息** - -**样例** - -- hdfs - -```json -{ - "namespace": "experiment", - "name": "breast_hetero_guest", - "engine": "HDFS", - "address": { - "name_node": "hdfs://fate-cluster", - "path": "/data/breast_hetero_guest.csv" - }, - "id_delimiter": ",", - "head": 1, - "partitions": 10 -} -``` - -- mysql - -```json -{ - "engine": "MYSQL", - "address": { - "user": "fate", - "passwd": "fate", - "host": "127.0.0.1", - "port": 3306, - "db": "experiment", - "name": "breast_hetero_guest" - }, - "namespace": "experiment", - "name": "breast_hetero_guest", - "head": 1, - "id_delimiter": ",", - "partitions": 10, - "id_column": "id", - "feature_column": "y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9" -} -``` - -- PATH - -```json -{ - "namespace": "xxx", - "name": "xxx", - "engine": "PATH", - "address": { - "path": "xxx" - } -} -``` -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -样例 - -```json -{ - "data": { - "namespace": "xxx", - "table_name": "xxx" - }, - "retcode": 0, - "retmsg": "success" -} -``` - - -### disable - -可通过table disable将表置为不可用状态 - -```bash -flow table disable [options] -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 必选 | 类型 | 说明 | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| table_name | `-t` |`--table-name` |是 | string | fate表名 | -| namespace | `-n` |`--namespace` | 是 |string | fate表命名空间 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -样例 - -```json -{ - "data": { - "namespace": "xxx", - "table_name": "xxx" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### enable - -可通过table enable将表置为可用状态 - -```bash -flow table enable [options] -``` - -**选项** - -| 参数 | 短格式 | 长格式 | 必选 | 类型 | 说明 | -| :-------- | :--- | :--- | :--- | :----- | -------------- | -| table_name | `-t` |`--table-name` |是 | string | fate表名 | -| namespace | `-n` |`--namespace` | 是 |string | fate表命名空间 | - - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -样例 - -```json -{ - "data": [{ - "namespace": "xxx", - "table_name": "xxx" - }], - "retcode": 0, - "retmsg": "success" -} -``` - -### disable-delete - -可通过disable-delete删除当前不可用的表 - -```bash -flow table disable-delete -``` - - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -样例 - -```json -{ - "data": [ - { - "namespace": "xxx", - "table_name": "xxx" - }, - { - "namespace": "xxx", - "table_name": "xxx" - } - ], - "retcode": 0, - "retmsg": "success" -} -``` \ No newline at end of file diff --git a/doc/cli/tag.md b/doc/cli/tag.md deleted file mode 100644 index 0f4605d30..000000000 --- a/doc/cli/tag.md +++ /dev/null @@ -1,89 +0,0 @@ -## Tag - -### create - -Creates a label. - -**Options** - -| number | parameters | short-format | long-format | required parameters | parameter description | -| ---- | ------------ | ------ | ------------ | -------- | -------- | -| 1 | tag_name | `-t` | `-tag-name` | yes | tag_name | -| 2 | tag_parameter_introduction | `-d` | `--tag-desc` | no | tag_introduction | - -**Example** - -``` bash -flow tag create -t tag1 -d "This is the parameter description of tag1." -flow tag create -t tag2 -``` - -### update - -Update the tag information. - -**Options** - -| number | parameters | short format | long format | required parameters | parameter description | -| ---- | ------------ | ------ | ---------------- | -------- | ---------- | -| 1 | tag_name | `-t` | `--tag-name` | yes | tag_name | -| 2 | new_tag_name | | `--new-tag-name` | no | new-tag-name | -| 3 | new_tag_desc | | `--new-tag-desc` | no | new tag introduction | - -**Example** - -``` bash -flow tag update -t tag1 --new-tag-name tag2 -flow tag update -t tag1 --new-tag-desc "This is the introduction of the new parameter." -``` - -### list - -Show the list of tags. - -**options** - -| number | parameters | short-format | long-format | required-parameters | parameter-introduction | -| ---- | ----- | ------ | --------- | -------- | ---------------------------- | -| 1 | limit | `-l` | `-limit` | no | Returns a limit on the number of results (default: 10) | - -**Example** - -``` bash -flow tag list -flow tag list -l 3 -``` - -### query - -Retrieve tags. - -**Options** - -| number | parameters | short-format | long-format | required parameters | parameter description | -| ---- | ---------- | ------ | -------------- | -------- | -------------------------------------- | -| 1 | tag_name | `-t` | `-tag-name` | yes | tag_name | -| 2 | with_model | | `-with-model` | no | If specified, information about models with this tag will be displayed | - -**Example** - -``` bash -flow tag query -t $TAG_NAME -flow tag query -t $TAG_NAME --with-model -``` - -### delete - -Delete the tag. - -**Options** - -| number | parameters | short-format | long-format | required-parameters | parameters introduction | -| ---- | -------- | ------ | ------------ | -------- | -------- -| 1 | tag_name | `-t` | `---tag-name` | yes | tag_name | - -**Example** - -``` bash -flow tag delete -t tag1 -``` diff --git a/doc/cli/tag.zh.md b/doc/cli/tag.zh.md deleted file mode 100644 index 9034c12c7..000000000 --- a/doc/cli/tag.zh.md +++ /dev/null @@ -1,89 +0,0 @@ -## Tag - -### create - -创建标签。 - -**选项** - -| 编号 | 参数 | 短格式 | 长格式 | 必要参数 | 参数介绍 | -| ---- | ------------ | ------ | ------------ | -------- | -------- | -| 1 | tag_name | `-t` | `--tag-name` | 是 | 标签名 | -| 2 | tag_参数介绍 | `-d` | `--tag-desc` | 否 | 标签介绍 | - -**样例** - -``` bash -flow tag create -t tag1 -d "This is the 参数介绍 of tag1." -flow tag create -t tag2 -``` - -### update - -更新标签信息。 - -**选项** - -| 编号 | 参数 | 短格式 | 长格式 | 必要参数 | 参数介绍 | -| ---- | ------------ | ------ | ---------------- | -------- | ---------- | -| 1 | tag_name | `-t` | `--tag-name` | 是 | 标签名 | -| 2 | new_tag_name | | `--new-tag-name` | 否 | 新标签名 | -| 3 | new_tag_desc | | `--new-tag-desc` | 否 | 新标签介绍 | - -**样例** - -``` bash -flow tag update -t tag1 --new-tag-name tag2 -flow tag update -t tag1 --new-tag-desc "This is the new 参数介绍." -``` - -### list - -展示标签列表。 - -**选项** - -| 编号 | 参数 | 短格式 | 长格式 | 必要参数 | 参数介绍 | -| ---- | ----- | ------ | --------- | -------- | ---------------------------- | -| 1 | limit | `-l` | `--limit` | 否 | 返回结果数量限制（默认：10） | - -**样例** - -``` bash -flow tag list -flow tag list -l 3 -``` - -### query - -检索标签。 - -**选项** - -| 编号 | 参数 | 短格式 | 长格式 | 必要参数 | 参数介绍 | -| ---- | ---------- | ------ | -------------- | -------- | -------------------------------------- | -| 1 | tag_name | `-t` | `--tag-name` | 是 | 标签名 | -| 2 | with_model | | `--with-model` | 否 | 如果指定，具有该标签的模型信息将被展示 | - -**样例** - -``` bash -flow tag query -t $TAG_NAME -flow tag query -t $TAG_NAME --with-model -``` - -### delete - -删除标签。 - -**选项** - -| 编号 | 参数 | 短格式 | 长格式 | 必要参数 | 参数介绍 | -| ---- | -------- | ------ | ------------ | -------- | -------- | -| 1 | tag_name | `-t` | `--tag-name` | 是 | 标签名 | - -**样例** - -``` bash -flow tag delete -t tag1 -``` diff --git a/doc/cli/task.md b/doc/cli/task.md deleted file mode 100644 index d7aad000f..000000000 --- a/doc/cli/task.md +++ /dev/null @@ -1,38 +0,0 @@ -## Task - -### query - -Retrieve Task information - -**Options** - -| number | parameters | short format | long format | required parameters | parameter description | -| ---- | -------------- | ------ | ------------------ | -------- | -------- | -| 1 | job_id | `-j` | `--job_id` | no | Job ID | -| 2 | role | `-r` | `--role` | no | role -| 3 | party_id | `-p` | `--party_id` | no | Party ID | -| 4 | component_name | `-cpn` | `--component_name` | no | component_name | -| 5 | status | `-s` | `--status` | No | Task status | - -**Example** - -``` bash -flow task query -j $JOB_ID -p 9999 -r guest -flow task query -cpn hetero_feature_binning_0 -s complete -``` - -### list - -Show the list of Tasks. -**Options** - -| number | parameters | short format | long format | required parameters | parameter description | -| ---- | ----- | ------ | --------- | -------- | ---------------------------- | -| 1 | limit | `-l` | `-limit` | no | Returns a limit on the number of results (default: 10) | - -**Example** - -``` bash -flow task list -flow task list -l 25 -``` diff --git a/doc/cli/task.zh.md b/doc/cli/task.zh.md deleted file mode 100644 index e7c94ee1d..000000000 --- a/doc/cli/task.zh.md +++ /dev/null @@ -1,38 +0,0 @@ -## Task - -### query - -检索Task信息 - -**选项** - -| 编号 | 参数 | 短格式 | 长格式 | 必要参数 | 参数介绍 | -| ---- | -------------- | ------ | ------------------ | -------- | -------- | -| 1 | job_id | `-j` | `--job_id` | 否 | Job ID | -| 2 | role | `-r` | `--role` | 否 | 角色 | -| 3 | party_id | `-p` | `--party_id` | 否 | Party ID | -| 4 | component_name | `-cpn` | `--component_name` | 否 | 组件名 | -| 5 | status | `-s` | `--status` | 否 | 任务状态 | - -**样例** - -``` bash -flow task query -j $JOB_ID -p 9999 -r guest -flow task query -cpn hetero_feature_binning_0 -s complete -``` - -### list - -展示Task列表。 -**选项** - -| 编号 | 参数 | 短格式 | 长格式 | 必要参数 | 参数介绍 | -| ---- | ----- | ------ | --------- | -------- | ---------------------------- | -| 1 | limit | `-l` | `--limit` | 否 | 返回结果数量限制（默认：10） | - -**样例** - -``` bash -flow task list -flow task list -l 25 -``` diff --git a/doc/cli/tracking.md b/doc/cli/tracking.md deleted file mode 100644 index 61525352b..000000000 --- a/doc/cli/tracking.md +++ /dev/null @@ -1,604 +0,0 @@ -## Tracking - -### metrics - -Get a list of all metrics names generated by a component task - -```bash -flow tracking metrics [options] -``` - -**Options** - -| parameter name | required | type | description | -| :--------------------- | :--- | :----- | ----------------------------- | -| -j, --job-id | yes | string | job-id | -| -r, --role | yes | string | participant-role | -| -p, --partyid | yes | string |participant-id | -| -cpn, --component-name | yes | string | Component name, consistent with that in job dsl | - -**Returns** - -| parameter-name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | -| data | dict | return data | - -**Example** - -```bash -flow tracking metrics -j 202111081618357358520 -r guest -p 9999 -cpn evaluation_0 -``` - -Output: - -```json -{ - "data": { - "train": [ - "hetero_lr_0", - "hetero_lr_0_ks_fpr", - "hetero_lr_0_ks_tpr", - "hetero_lr_0_lift", - "hetero_lr_0_gain", - "hetero_lr_0_accuracy", - "hetero_lr_0_precision", - "hetero_lr_0_recall", - "hetero_lr_0_roc", - "hetero_lr_0_confusion_mat", - "hetero_lr_0_f1_score", - "hetero_lr_0_quantile_pr" - ] - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### metric-all - -Get all the output metrics for a component task - -```bash -flow tracking metric-all [options] -``` - -**Options** - -| parameter-name | required | type | description | -| :--------------------- | :--- | :----- | ----------------------------- | -| -j, --job-id | yes | string | job-id | -| -r, --role | yes | string | participant-role | -| -p, --partyid | yes | string |participant-id | -| -cpn, --component-name | yes | string | Component name, consistent with that in job dsl | - -**Returns** - -| parameter-name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | -| data | dict | return data | -| jobId | string | job id | - -**Example** - -```bash -flow tracking metric-all -j 202111081618357358520 -r guest -p 9999 -cpn evaluation_0 -``` - -Output (limited space, only some of the metric data is shown and some values are omitted in the middle of the array type data): - -```json -{ - "data": { - "train": { - "hetero_lr_0": { - "data": [ - [ - "auc", - 0.293893 - ], - [ - "ks", - 0.0 - ] - ], - "meta": { - "metric_type": "EVALUATION_SUMMARY", - "name": "hetero_lr_0" - } - }, - "hetero_lr_0_accuracy": { - "data": [ - [ - 0.0, - 0.372583 - ], - [ - 0.99, - 0.616872 - ] - ], - "meta": { - "curve_name": "hetero_lr_0", - "metric_type": "ACCURACY_EVALUATION", - "name": "hetero_lr_0_accuracy", - "thresholds": [ - 0.999471, - 0.002577 - ] - } - }, - "hetero_lr_0_confusion_mat": { - "data": [], - "meta": { - "fn": [ - 357, - 0 - ], - "fp": [ - 0, - 212 - ], - "metric_type": "CONFUSION_MAT", - "name": "hetero_lr_0_confusion_mat", - "thresholds": [ - 0.999471, - 0.0 - ], - "tn": [ - 212, - 0 - ], - "tp": [ - 0, - 357 - ] - } - } - } - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### parameters - -After the job is submitted, the system resolves the actual component task parameters based on the component_parameters in the job conf combined with the system default component parameters - -```bash -flow tracking parameters [options] -``` - -**Options** - -| parameter_name | required | type | description | -| :--------------------- | :--- | :----- | ----------------------------- | -| -j, --job-id | yes | string | job-id | -| -r, --role | yes | string | participant-role | -| -p, --partyid | yes | string |participant-id | -| -cpn, --component-name | yes | string | Component name, consistent with that in job dsl | - - -**Returns** - -| parameter-name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | -| data | dict | return data | -| jobId | string | job id | - -**Example** - -```bash -flow tracking parameters -j 202111081618357358520 -r guest -p 9999 -cpn hetero_lr_0 -``` - -Output: - -```json -{ - "data": { - "ComponentParam": { - "_feeded_deprecated_params": [], - "_is_raw_conf": false, - "_name": "HeteroLR#hetero_lr_0", - "_user_feeded_params": [ - "batch_size", - "penalty", - "max_iter", - "learning_rate", - "init_param", - "optimizer", - "init_param.init_method", - "alpha" - ], - "alpha": 0.01, - "batch_size": 320, - "callback_param": { - "callbacks": [], - "early_stopping_rounds": null, - "metrics": [], - "save_freq": 1, - "use_first_metric_only": false, - "validation_freqs": null - }, - "cv_param": { - "history_value_type": "score", - "mode": "hetero", - "n_splits": 5, - "need_cv": false, - "output_fold_history": true, - "random_seed": 1, - "role": "guest", - "shuffle": true - }, - "decay": 1, - "decay_sqrt": true, - "early_stop": "diff", - "early_stopping_rounds": null, - "encrypt_param": { - "key_length": 1024, - "method": "Paillier" - }, - "encrypted_mode_calculator_param": { - "mode": "strict", - "re_encrypted_rate": 1 - }, - "floating_point_precision": 23, - "init_param": { - "fit_intercept": true, - "init_const": 1, - "init_method": "random_uniform", - "random_seed": null - }, - "learning_rate": 0.15, - "max_iter": 3, - "metrics": [ - "auc", - "ks" - ], - "multi_class": "ovr", - "optimizer": "rmsprop", - "penalty": "L2", - "predict_param": { - "threshold": 0.5 - }, - "sqn_param": { - "memory_M": 5, - "random_seed": null, - "sample_size": 5000, - "update_interval_L": 3 - }, - "stepwise_param": { - "direction": "both", - "max_step": 10, - "mode": "hetero", - "need_stepwise": false, - "nvmax": null, - "nvmin": 2, - "role": "guest", - "score_name": "AIC" - }, - "tol": 0.0001, - "use_first_metric_only": false, - "validation_freqs": null - }, - "module": "HeteroLR" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### output-data - -Get the component output - -```bash -flow tracking output-data [options] -``` - -**options** - -| parameter-name | required | type | description | -| :--------------------- | :--- | :----- | ----------------------------- | -| -j, --job-id | yes | string | job-id | -| -r, --role | yes | string | participant-role | -| -p, --partyid | yes | string |participant-id | -| -cpn, --component-name | yes | string | Component name, consistent with that in job dsl | -| -o, --output-path | yes | string | Path to output data | - -**Returns** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | Return code | -| retmsg | string | return message | -| data | dict | return data | -| jobId | string | job id | - -**Example** - -```bash -flow tracking output-data -j 202111081618357358520 -r guest -p 9999 -cpn hetero_lr_0 -o . / -``` - -Output : - -```json -{ - "retcode": 0, - "directory": "$FATE_PROJECT_BASE/job_202111081618357358520_hetero_lr_0_guest_9999_output_data", - "retmsg": "Download successfully, please check $FATE_PROJECT_BASE/job_202111081618357358520_hetero_lr_0_guest_9999_output_data directory " -} -``` - -### output-data-table - -Get the output data table name of the component - -```bash -flow tracking output-data-table [options] -``` - -**options** - -| parameter-name | required | type | description | -| :--------------------- | :--- | :----- | ----------------------------- | -| -j, --job-id | yes | string | job-id | -| -r, --role | yes | string | participant-role | -| -p, --partyid | yes | string |participant-id | -| -cpn, --component-name | yes | string | Component name, consistent with that in job dsl | - -**Returns** - -| parameter-name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | -| data | dict | return data | -| jobId | string | job id | - -**Example** - -```bash -flow tracking output-data-table -j 202111081618357358520 -r guest -p 9999 -cpn hetero_lr_0 -``` - -output: - -```json -{ - "data": [ - { - "data_name": "train", - "table_name": "9688fa00406c11ecbd0bacde48001122", - "table_namespace": "output_data_202111081618357358520_hetero_lr_0_0" - } - ], - "retcode": 0, - "retmsg": "success" -} -``` - -### output-model - -Get the output model of a component task - -```bash -flow tracking output-model [options] -``` - -**options** - -| parameter-name | required | type | description | -| :--------------------- | :--- | :----- | ----------------------------- | -| -j, --job-id | yes | string | job-id | -| -r, --role | yes | string | participant-role | -| -p, --partyid | yes | string |participant-id | -| -cpn, --component-name | yes | string | Component name, consistent with that in job dsl | - -**Returns** - -| parameter-name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | -| data | dict | return data | -| jobId | string | job id | - -**Example** - -```bash -flow tracking output-model -j 202111081618357358520 -r guest -p 9999 -cpn hetero_lr_0 -``` - -Output: - -```json -{ - "data": { - "bestIteration": -1, - "encryptedWeight": {}, - "header": [ - "x0", - "x1", - "x2", - "x3", - "x4", - "x5", - "x6", - "x7", - "x8", - "x9" - ], - "intercept": 0.24451607054764884, - "isConverged": false, - "iters": 3, - "lossHistory": [], - "needOneVsRest": false, - "weight": { - "x0": 0.04639947589856569, - "x1": 0.19899685467216902, - "x2": -0.18133550931649306, - "x3": 0.44928868756862206, - "x4": 0.05285905125502288, - "x5": 0.319187932844076, - "x6": 0.42578983446194013, - "x7": -0.025765956309895477, - "x8": -0.3699194462271593, - "x9": -0.1212094750908295 - } - }, - "meta": { - "meta_data": { - "alpha": 0.01, - "batchSize": "320", - "earlyStop": "diff", - "fitIntercept": true, - "learningRate": 0.15, - "maxIter": "3", - "needOneVsRest": false, - "optimizer": "rmsprop", - "partyWeight": 0.0, - "penalty": "L2", - "reEncryptBatches": "0", - "revealStrategy": "", - "tol": 0.0001 - }, - "module_name": "HeteroLR" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### get-summary - -Each component allows to set some summary information for easy observation and analysis - -```bash -flow tracking get-summary [options] -``` - -**Options** - -| parameter-name | required | type | description | -| :--------------------- | :--- | :----- | ----------------------------- | -| -j, --job-id | yes | string | job-id | -| -r, --role | yes | string | participant-role | -| -p, --partyid | yes | string |participant-id | -| -cpn, --component-name | yes | string | Component name, consistent with that in job dsl | - -**Returns** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | -| data | dict | return data | -| jobId | string | job id | - -**Example** - -```bash -flow tracking get-summary -j 202111081618357358520 -r guest -p 9999 -cpn hetero_lr_0 -``` - -Output: - -```json -{ - "data": { - "best_iteration": -1, - "coef": { - "x0": 0.04639947589856569, - "x1": 0.19899685467216902, - "x2": -0.18133550931649306, - "x3": 0.44928868756862206, - "x4": 0.05285905125502288, - "x5": 0.319187932844076, - "x6": 0.42578983446194013, - "x7": -0.025765956309895477, - "x8": -0.3699194462271593, - "x9": -0.1212094750908295 - }, - "intercept": 0.24451607054764884, - "is_converged": false, - "one_vs_rest": false - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### tracking-source - -For querying the parent and source tables of a table - -```bash -flow table tracking-source [options] -``` - -**Options** - -| parameter-name | required | type | description | -| :-------- | :--- | :----- | -------------- | -| name | yes | string | fate table name | -| namespace | yes | string | fate table namespace | - -**Returns** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | -| data | object | return data | - -**Example** - -```json -{ - "data": [{"parent_table_name": "61210fa23c8d11ec849a5254004fdc71", "parent_table_namespace": "output_data_202111031759294631020_hetero _lr_0_0", "source_table_name": "breast_hetero_guest", "source_table_namespace": "experiment"}], - "retcode": 0, - "retmsg": "success" -} -``` - -### tracking-job - -For querying the usage of a particular table - -```bash -flow table tracking-job [options] -``` - -**Options** - -| parameter name | required | type | description | -| :-------- | :--- | :----- | -------------- | -| name | yes | string | fate table name | -| namespace | yes | string | fate table namespace | - -**Returns** - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | -| data | object | return data | - -**Example** - -```json -{ - "data": {"count":2, "jobs":["202111052115375327830", "202111031816501123160"]}, - "retcode": 0, - "retmsg": "success" -} -``` diff --git a/doc/cli/tracking.zh.md b/doc/cli/tracking.zh.md deleted file mode 100644 index 2b5972a5e..000000000 --- a/doc/cli/tracking.zh.md +++ /dev/null @@ -1,604 +0,0 @@ -## Tracking - -### metrics - -获取某个组件任务产生的所有指标名称列表 - -```bash -flow tracking metrics [options] -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :--------------------- | :--- | :----- | ----------------------------- | -| -j, --job-id | 是 | string | 作业id | -| -r, --role | 是 | string | 参与角色 | -| -p, --partyid | 是 | string | 参与方id | -| -cpn, --component-name | 是 | string | 组件名，与job dsl中的保持一致 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | dict | 返回数据 | - -**样例** - -```bash -flow tracking metrics -j 202111081618357358520 -r guest -p 9999 -cpn evaluation_0 -``` - -输出: - -```json -{ - "data": { - "train": [ - "hetero_lr_0", - "hetero_lr_0_ks_fpr", - "hetero_lr_0_ks_tpr", - "hetero_lr_0_lift", - "hetero_lr_0_gain", - "hetero_lr_0_accuracy", - "hetero_lr_0_precision", - "hetero_lr_0_recall", - "hetero_lr_0_roc", - "hetero_lr_0_confusion_mat", - "hetero_lr_0_f1_score", - "hetero_lr_0_quantile_pr" - ] - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### metric-all - -获取组件任务的所有输出指标 - -```bash -flow tracking metric-all [options] -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :--------------------- | :--- | :----- | ----------------------------- | -| -j, --job-id | 是 | string | 作业id | -| -r, --role | 是 | string | 参与角色 | -| -p, --partyid | 是 | string | 参与方id | -| -cpn, --component-name | 是 | string | 组件名，与job dsl中的保持一致 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | dict | 返回数据 | -| jobId | string | 作业id | - -**样例** - -```bash -flow tracking metric-all -j 202111081618357358520 -r guest -p 9999 -cpn evaluation_0 -``` - -输出(篇幅有限，仅显示部分指标的数据且数组型数据中间省略了一些值): - -```json -{ - "data": { - "train": { - "hetero_lr_0": { - "data": [ - [ - "auc", - 0.293893 - ], - [ - "ks", - 0.0 - ] - ], - "meta": { - "metric_type": "EVALUATION_SUMMARY", - "name": "hetero_lr_0" - } - }, - "hetero_lr_0_accuracy": { - "data": [ - [ - 0.0, - 0.372583 - ], - [ - 0.99, - 0.616872 - ] - ], - "meta": { - "curve_name": "hetero_lr_0", - "metric_type": "ACCURACY_EVALUATION", - "name": "hetero_lr_0_accuracy", - "thresholds": [ - 0.999471, - 0.002577 - ] - } - }, - "hetero_lr_0_confusion_mat": { - "data": [], - "meta": { - "fn": [ - 357, - 0 - ], - "fp": [ - 0, - 212 - ], - "metric_type": "CONFUSION_MAT", - "name": "hetero_lr_0_confusion_mat", - "thresholds": [ - 0.999471, - 0.0 - ], - "tn": [ - 212, - 0 - ], - "tp": [ - 0, - 357 - ] - } - } - } - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### parameters - -提交作业后，系统依据job conf中的component_parameters结合系统默认组件参数，最终解析得到的实际组件任务运行参数 - -```bash -flow tracking parameters [options] -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :--------------------- | :--- | :----- | ----------------------------- | -| -j, --job-id | 是 | string | 作业id | -| -r, --role | 是 | string | 参与角色 | -| -p, --partyid | 是 | string | 参与方id | -| -cpn, --component-name | 是 | string | 组件名，与job dsl中的保持一致 | - - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | dict | 返回数据 | -| jobId | string | 作业id | - -**样例** - -```bash -flow tracking parameters -j 202111081618357358520 -r guest -p 9999 -cpn hetero_lr_0 -``` - -输出: - -```json -{ - "data": { - "ComponentParam": { - "_feeded_deprecated_params": [], - "_is_raw_conf": false, - "_name": "HeteroLR#hetero_lr_0", - "_user_feeded_params": [ - "batch_size", - "penalty", - "max_iter", - "learning_rate", - "init_param", - "optimizer", - "init_param.init_method", - "alpha" - ], - "alpha": 0.01, - "batch_size": 320, - "callback_param": { - "callbacks": [], - "early_stopping_rounds": null, - "metrics": [], - "save_freq": 1, - "use_first_metric_only": false, - "validation_freqs": null - }, - "cv_param": { - "history_value_type": "score", - "mode": "hetero", - "n_splits": 5, - "need_cv": false, - "output_fold_history": true, - "random_seed": 1, - "role": "guest", - "shuffle": true - }, - "decay": 1, - "decay_sqrt": true, - "early_stop": "diff", - "early_stopping_rounds": null, - "encrypt_param": { - "key_length": 1024, - "method": "Paillier" - }, - "encrypted_mode_calculator_param": { - "mode": "strict", - "re_encrypted_rate": 1 - }, - "floating_point_precision": 23, - "init_param": { - "fit_intercept": true, - "init_const": 1, - "init_method": "random_uniform", - "random_seed": null - }, - "learning_rate": 0.15, - "max_iter": 3, - "metrics": [ - "auc", - "ks" - ], - "multi_class": "ovr", - "optimizer": "rmsprop", - "penalty": "L2", - "predict_param": { - "threshold": 0.5 - }, - "sqn_param": { - "memory_M": 5, - "random_seed": null, - "sample_size": 5000, - "update_interval_L": 3 - }, - "stepwise_param": { - "direction": "both", - "max_step": 10, - "mode": "hetero", - "need_stepwise": false, - "nvmax": null, - "nvmin": 2, - "role": "guest", - "score_name": "AIC" - }, - "tol": 0.0001, - "use_first_metric_only": false, - "validation_freqs": null - }, - "module": "HeteroLR" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### output-data - -获取组件输出 - -```bash -flow tracking output-data [options] -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :--------------------- | :--- | :----- | ----------------------------- | -| -j, --job-id | 是 | string | 作业id | -| -r, --role | 是 | string | 参与角色 | -| -p, --partyid | 是 | string | 参与方id | -| -cpn, --component-name | 是 | string | 组件名，与job dsl中的保持一致 | -| -o, --output-path | 是 | string | 输出数据的存放路径 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | dict | 返回数据 | -| jobId | string | 作业id | - -**样例** - -```bash -flow tracking output-data -j 202111081618357358520 -r guest -p 9999 -cpn hetero_lr_0 -o ./ -``` - -输出: - -```json -{ - "retcode": 0, - "directory": "$FATE_PROJECT_BASE/job_202111081618357358520_hetero_lr_0_guest_9999_output_data", - "retmsg": "Download successfully, please check $FATE_PROJECT_BASE/job_202111081618357358520_hetero_lr_0_guest_9999_output_data directory" -} -``` - -### output-data-table - -获取组件的输出数据表名 - -```bash -flow tracking output-data-table [options] -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :--------------------- | :--- | :----- | ----------------------------- | -| -j, --job-id | 是 | string | 作业id | -| -r, --role | 是 | string | 参与角色 | -| -p, --partyid | 是 | string | 参与方id | -| -cpn, --component-name | 是 | string | 组件名，与job dsl中的保持一致 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | dict | 返回数据 | -| jobId | string | 作业id | - -**样例** - -```bash -flow tracking output-data-table -j 202111081618357358520 -r guest -p 9999 -cpn hetero_lr_0 -``` - -输出: - -```json -{ - "data": [ - { - "data_name": "train", - "table_name": "9688fa00406c11ecbd0bacde48001122", - "table_namespace": "output_data_202111081618357358520_hetero_lr_0_0" - } - ], - "retcode": 0, - "retmsg": "success" -} -``` - -### output-model - -获取某个组件任务的输出模型 - -```bash -flow tracking output-model [options] -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :--------------------- | :--- | :----- | ----------------------------- | -| -j, --job-id | 是 | string | 作业id | -| -r, --role | 是 | string | 参与角色 | -| -p, --partyid | 是 | string | 参与方id | -| -cpn, --component-name | 是 | string | 组件名，与job dsl中的保持一致 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | dict | 返回数据 | -| jobId | string | 作业id | - -**样例** - -```bash -flow tracking output-model -j 202111081618357358520 -r guest -p 9999 -cpn hetero_lr_0 -``` - -输出: - -```json -{ - "data": { - "bestIteration": -1, - "encryptedWeight": {}, - "header": [ - "x0", - "x1", - "x2", - "x3", - "x4", - "x5", - "x6", - "x7", - "x8", - "x9" - ], - "intercept": 0.24451607054764884, - "isConverged": false, - "iters": 3, - "lossHistory": [], - "needOneVsRest": false, - "weight": { - "x0": 0.04639947589856569, - "x1": 0.19899685467216902, - "x2": -0.18133550931649306, - "x3": 0.44928868756862206, - "x4": 0.05285905125502288, - "x5": 0.319187932844076, - "x6": 0.42578983446194013, - "x7": -0.025765956309895477, - "x8": -0.3699194462271593, - "x9": -0.1212094750908295 - } - }, - "meta": { - "meta_data": { - "alpha": 0.01, - "batchSize": "320", - "earlyStop": "diff", - "fitIntercept": true, - "learningRate": 0.15, - "maxIter": "3", - "needOneVsRest": false, - "optimizer": "rmsprop", - "partyWeight": 0.0, - "penalty": "L2", - "reEncryptBatches": "0", - "revealStrategy": "", - "tol": 0.0001 - }, - "module_name": "HeteroLR" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### get-summary - -每个组件允许设置一些摘要信息，便于观察分析 - -```bash -flow tracking get-summary [options] -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :--------------------- | :--- | :----- | ----------------------------- | -| -j, --job-id | 是 | string | 作业id | -| -r, --role | 是 | string | 参与角色 | -| -p, --partyid | 是 | string | 参与方id | -| -cpn, --component-name | 是 | string | 组件名，与job dsl中的保持一致 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | dict | 返回数据 | -| jobId | string | 作业id | - -**样例** - -```bash -flow tracking get-summary -j 202111081618357358520 -r guest -p 9999 -cpn hetero_lr_0 -``` - -输出: - -```json -{ - "data": { - "best_iteration": -1, - "coef": { - "x0": 0.04639947589856569, - "x1": 0.19899685467216902, - "x2": -0.18133550931649306, - "x3": 0.44928868756862206, - "x4": 0.05285905125502288, - "x5": 0.319187932844076, - "x6": 0.42578983446194013, - "x7": -0.025765956309895477, - "x8": -0.3699194462271593, - "x9": -0.1212094750908295 - }, - "intercept": 0.24451607054764884, - "is_converged": false, - "one_vs_rest": false - }, - "retcode": 0, - "retmsg": "success" -} -``` - -### tracking-source - -用于查询某张表的父表及源表 - -```bash -flow table tracking-source [options] -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :-------- | :--- | :----- | -------------- | -| name | 是 | string | fate表名 | -| namespace | 是 | string | fate表命名空间 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -**样例** - -```json -{ - "data": [{"parent_table_name": "61210fa23c8d11ec849a5254004fdc71", "parent_table_namespace": "output_data_202111031759294631020_hetero_lr_0_0", "source_table_name": "breast_hetero_guest", "source_table_namespace": "experiment"}], - "retcode": 0, - "retmsg": "success" -} -``` - -### tracking-job - -用于查询某张表的使用情况 - -```bash -flow table tracking-job [options] -``` - -**选项** - -| 参数名 | 必选 | 类型 | 说明 | -| :-------- | :--- | :----- | -------------- | -| name | 是 | string | fate表名 | -| namespace | 是 | string | fate表命名空间 | - -**返回** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | object | 返回数据 | - -**样例** - -```json -{ - "data": {"count":2,"job":["202111052115375327830", "202111031816501123160"]}, - "retcode": 0, - "retmsg": "success" -} -``` diff --git a/doc/configuration_instruction.md b/doc/configuration_instruction.md deleted file mode 100644 index a4c7bba87..000000000 --- a/doc/configuration_instruction.md +++ /dev/null @@ -1,412 +0,0 @@ -# Configuration Instructions - -## 1. Description - -Contains the general configuration of the `FATE project` and the configuration of each subsystem - -## 2. Global configuration - -- Path: `${FATE_PROJECT_BASE}/conf/server_conf.yaml` -- Description: Commonly used configuration, generally needed to determine when deploying -- Note: Configuration items that are not listed below in the configuration file are internal system parameters and are not recommended to be modified - -```yaml -# If FATEFlow uses the registry, FATEFlow will register the FATEFlow Server address and the published model download address to the registry for the online system FATEServing; it will also get the FATEServing address from the registry. -use_registry: false -# Whether to enable higher security serialization mode -use_deserialize_safe_module: false -dependent_distribution: false -# party id： required for site authentication -party_id: -# Hook module configuration -hook_module: - # Client authentication hooks - client_authentication: fate_flow.hook.flow.client_authentication - # site-side authentication hooks - site_authentication: fate_flow.hook.flow.site_authentication - # Permission authentication hooks - permission: fate_flow.hook.flow.permission -# In addition to using flow's hooks for authentication and authentication, we also support authentication and authentication interfaces registered with third-party services -# The name of the service registered by the third-party authentication and authentication service -hook_server_name: -# Authentication -authentication: - # Client authentication configuration - client: - # Client authentication switch - switch: false - http_app_key: - http_secret_key: - # Site authentication configuration - site: - # Authentication switch - switch: false -# Authentication -permission: - # Authentication switch - switch: false - # Component authentication switch - component: false - # Data set authentication switch - dataset: false -fateflow: - # you must set real ip address, 127.0.0.1 and 0.0.0.0 is not supported - host: 127.0.0.1 - http_port: 9380 - grpc_port: 9360 - # The nginx address needs to be configured for high availability - nginx: - host: - http_port: - grpc_port: - # use random instance_id instead of {host}:{http_port} - random_instance_id: false - # support rollsite/nginx/fateflow as a coordination proxy - # rollsite support fate on eggroll, use grpc protocol - # nginx support fate on eggroll and fate on spark, use http or grpc protocol, default is http - # fateflow support fate on eggroll and fate on spark, use http protocol, but not support exchange network mode - - # format(proxy: rollsite) means rollsite use the rollsite configuration of fate_one_eggroll and nginx use the nginx configuration of fate_one_spark - # you also can customize the config like this(set fateflow of the opposite party as proxy): - # proxy: - # name: fateflow - # host: xx - # http_port: xx - # grpc_port: xx - proxy: rollsite - # support default/http/grpc - protocol: default -database: - name: fate_flow - user: fate - passwd: fate - host: 127.0.0.1 - port: 3306 - max_connections: 100 - stale_timeout: 30 -# The registry address and its authentication parameters -zookeeper: - hosts: - - 127.0.0.1:2181 - use_acl: false - user: fate - password: fate -# engine services -default_engines: - computing: standalone - federation: standalone - storage: standalone -fate_on_standalone: - standalone: - cores_per_node: 20 - nodes: 1 -fate_on_eggroll: - clustermanager: - # CPU cores of the machine where eggroll nodemanager service is running - cores_per_node: 16 - # the number of eggroll nodemanager machine - nodes: 1 - rollsite: - host: 127.0.0.1 - port: 9370 -fate_on_spark: - spark: - # default use SPARK_HOME environment variable - home: - cores_per_node: 20 - nodes: 2 - linkis_spark: - cores_per_node: 20 - nodes: 2 - host: 127.0.0.1 - port: 9001 - token_code: MLSS - python_path: /data/projects/fate/python - hive: - host: 127.0.0.1 - port: 10000 - auth_mechanism: - username: - password: - linkis_hive: - host: 127.0.0.1 - port: 9001 - hdfs: - name_node: hdfs://fate-cluster - # default / - path_prefix: - rabbitmq: - host: 192.168.0.4 - mng_port: 12345 - port: 5672 - user: fate - password: fate - # default conf/rabbitmq_route_table.yaml - route_table: - pulsar: - host: 192.168.0.5 - port: 6650 - mng_port: 8080 - cluster: standalone - # all parties should use a same tenant - tenant: fl-tenant - # message ttl in minutes - topic_ttl: 5 - # default conf/pulsar_route_table.yaml - route_table: - nginx: - host: 127.0.0.1 - http_port: 9300 - grpc_port: 9310 -# external services -fateboard: - host: 127.0.0.1 - port: 8080 - -# on API `/model/load` and `/model/load/do` -# automatic upload models to the model store if it exists locally but does not exist in the model storage -# or download models from the model store if it does not exist locally but exists in the model storage -# this config will not affect API `/model/store` or `/model/restore` -enable_model_store: false -# default address for export model -model_store_address: - # use mysql as the model store engine -# storage: mysql -# database: fate_model -# user: fate -# password: fate -# host: 127.0.0.1 -# port: 3306 - # other optional configs send to the engine -# max_connections: 10 -# stale_timeout: 10 - # use redis as the model store engine -# storage: redis -# host: 127.0.0.1 -# port: 6379 -# db: 0 -# password: - # the expiry time of keys, in seconds. defaults None (no expiry time) -# ex: - # use tencent cos as model store engine - storage: tencent_cos - Region: - SecretId: - SecretKey: - Bucket: - -# The address of the FATE Serving Server needs to be configured if the registry is not used -servings: - hosts: - - 127.0.0.1:8000 -fatemanager: - host: 127.0.0.1 - port: 8001 - federatedId: 0 - -``` - -## 3. FATE Flow Configuration - -### 3.1 FATE Flow Server Configuration - -- Path: `${FATE_FLOW_BASE}/python/fate_flow/settings.py` -- Description: Advanced configuration, generally no changes are needed -- Note: Configuration items that are not listed below in the configuration file are internal system parameters and are not recommended to be modified - -```python -# Thread pool size of grpc server used by FATE Flow Server for multiparty FATE Flow Server communication, not set default equal to the number of CPU cores of the machine -GRPC_SERVER_MAX_WORKERS = None - -# Switch -# The upload data interface gets data from the client by default, this value can be configured at the time of the interface call using use_local_data -UPLOAD_DATA_FROM_CLIENT = True -# Whether to enable multi-party communication authentication, need to be used with FATE Cloud -CHECK_NODES_IDENTITY = False -# Whether to enable the resource authentication function, need to use with FATE Cloud -USE_AUTHENTICATION = False -# Resource privileges granted by default -PRIVILEGE_COMMAND_WHITELIST = [] -``` - -### 3.2 FATE Flow Default Job Configuration - -- Path: `${FATE_FLOW_BASE}/conf/job_default_config.yaml` -- Description: Advanced configuration, generally no changes are needed -- Note: Configuration items that are not listed below in the configuration file are internal system parameters and are not recommended to be modified -- Take effect: use flow server reload or restart fate flow server - -```yaml -# component provider, relative path to get_fate_python_directory -default_component_provider_path: federatedml - -# resource -# total_cores_overweight_percent -total_cores_overweight_percent: 1 # 1 means no overweight -total_memory_overweight_percent: 1 # 1 means no overweight -# Default task parallelism per job, you can configure a custom value using job_parameters:task_parallelism when submitting the job configuration -task_parallelism: 1 -# The default number of CPU cores per task per job, which can be configured using job_parameters:task_cores when submitting the job configuration -task_cores: 4 -# This configuration does not take effect as memory resources are not supported for scheduling at the moment -task_memory: 0 # mb -# The ratio of the maximum number of CPU cores allowed for a job to the total number of resources, e.g., if the total resources are 10 and the value is 0.5, then a job is allowed to request up to 5 CPUs, i.e., task_cores * task_parallelism <= 10 * 0.5 -max_cores_percent_per_job: 1 # 1 means total - -# scheduling -# Default job execution timeout, you can configure a custom value using job_parameters:timeout when submitting the job configuration -job_timeout: 259200 # s -# Timeout for communication when sending cross-participant scheduling commands or status -remote_request_timeout: 30000 # ms -# Number of retries to send cross-participant scheduling commands or status -federated_command_trys: 3 -end_status_job_scheduling_time_limit: 300000 # ms -end_status_job_scheduling_updates: 1 -# Default number of auto retries, you can configure a custom value using job_parameters:auto_retries when submitting the job configuration -auto_retries: 0 -# Default retry interval -auto_retry_delay: 1 #seconds -# Default multiparty status collection method, supports PULL and PUSH; you can also specify the current job collection mode in the job configuration -federated_status_collect_type: PUSH - -# upload -upload_max_bytes: 104857600 # bytes - -#component output -output_data_summary_count_limit: 100 -``` - -## 4. FATE Board Configuration - -- Path: `${FATE_BOARD_BASE}/conf/application.properties` -- Description: Commonly used configuration, generally needed to determine when deploying -- Note: Configuration items that are not listed below in the configuration file are internal system parameters and are not recommended to be modified - -```properties -# Service listening port -server.port=8080 -# fateflow address, referring to the http port address of fateflow -fateflow.url==http://127.0.0.1:9380 -# db address, same as the above global configuration service_conf.yaml inside the database configuration -fateboard.datasource.jdbc-url=jdbc:mysql://localhost:3306/fate_flow?characterEncoding=utf8&characterSetResults=utf8&autoReconnect= true&failOverReadOnly=false&serverTimezone=GMT%2B8 -# db configuration, same as the above global configuration service_conf.yaml inside the database configuration -fateboard.datasource.username= -# db configuration, same as the above global configuration service_conf.yaml inside the database configuration -fateboard.datasource.password= -server.tomcat.max-threads=1000 -server.tomcat.max-connections=20000 -spring.servlet.multipart.max-file-size=10MB -spring.servlet.multipart.max-request-size=100MB -# Administrator account configuration -server.board.login.username=admin -server.board.login.password=admin -server.ssl.key-store=classpath: -server.ssl.key-store-password= -server.ssl.key-password= -server.ssl.key-alias= -# When fateflo server enables api access authentication, you need to configure -HTTP_APP_KEY= -HTTP_SECRET_KEY= -``` - -## 5. EggRoll - -### 5.1 System configuration - -- Path: `${EGGROLL_HOME}/conf/eggroll.properties` -- Description: Commonly used configuration, generally needed to determine when deploying -- Note: Configuration items that are not listed below in the configuration file are internal system parameters and are not recommended to be modified - -```properties -[eggroll] -# core -# MySQL connection configuration, generally required for production applications -eggroll.resourcemanager.clustermanager.jdbc.driver.class.name=com.mysql.cj.jdbc. -# MySQL connection configuration, generally required for production applications -eggroll.resourcemanager.clustermanager.jdbc.url=jdbc:mysql://localhost:3306/eggroll_meta?useSSL=false&serverTimezone=UTC& characterEncoding=utf8&allowPublicKeyRetrieval=true -# Connect to MySQL account, this configuration is required for general production applications -eggroll.resourcemanager.clustermanager.jdbc.username= -# Connect to MySQL password, generally required for production applications -eggroll.resourcemanager.clustermanager.jdbc.password= - -# Data storage directory -eggroll.data.dir=data/ -# Log storage directory -eggroll.logs.dir=logs/ -eggroll.resourcemanager.clustermanager.host=127.0.0.1 -eggroll.resourcemanager.clustermanager.port=4670 -eggroll.resourcemanager.nodemanager.port=4670 - -# python path -eggroll.resourcemanager.bootstrap.eggg_pair.venv= -# pythonpath, usually you need to specify the python directory of eggroll and the python directory of fate -eggroll.resourcemanager.bootstrap.eggg_pair.pythonpath=python - -# java path -eggroll.resourcemanager.bootstrap.eggg_frame.javahome= -# java service startup parameters, no special needs, no need to configure -eggroll.resourcemanager.bootstrap.eggg_frame.jvm.options= -# grpc connection hold time for multi-party communication -eggroll.core.grpc.channel.keepalive.timeout.sec=20 - -# session -# Number of computing processes started per nodemanager in an eggroll session; replaced by the default parameters of the fate flow if using fate for committing tasks -eggroll.session.processors.per.node=4 - -# rollsite -eggroll.rollsite.coordinator=webank -eggroll.rollsite.host=127.0.0.1 -eggroll.rollsite.port=9370 -eggroll.rollsite.party.id=10001 -eggroll.rollsite.route.table.path=conf/route_table.json - -eggroll.rollsite.push.max.retry=3 -eggroll.rollsite.push.long.retry=2 -eggroll.rollsite.push.batches.per.stream=10 -eggroll.rollsite.adapter.sendbuf.size=100000 -``` - -### 5.2 Routing table configuration - -- Path: `${EGGROLL_HOME}/conf/route_table.json` -- Description: Commonly used configuration, generally needed to determine when deploying - - The routing table is mainly divided into two levels - - The first level indicates the site, if the corresponding target site configuration is not found, then use **default** - - The second level represents the service, if you can not find the corresponding target service, then use **default** - - The second level, usually set **default** as the address of our **rollsite** service, and **fateflow** as the grpc address of our **fate flow server** service - -```json -{ - "route_table": - { - "10001": - { - "default":[ - { - "port": 9370, - "ip": "127.0.0.1" - } - ], - "fateflow":[ - { - "port": 9360, - "ip": "127.0.0.1" - } - ] - }, - "10002": - { - "default":[ - { - "port": 9470, - "ip": "127.0.0.1" - } - ] - } - }, - "permission": - { - "default_allow": true - } -} -``` diff --git a/doc/configuration_instruction.zh.md b/doc/configuration_instruction.zh.md deleted file mode 100644 index 4dd2038bb..000000000 --- a/doc/configuration_instruction.zh.md +++ /dev/null @@ -1,419 +0,0 @@ -# 配置说明 - -## 1. 说明 - -包含`FATE项目`总配置以及各个子系统的配置 - -## 2. 全局配置 - -- 路径：`${FATE_PROJECT_BASE}/conf/server_conf.yaml` -- 说明：常用配置，一般部署时需要确定 -- 注意：配置文件中未被列举如下的配置项属于系统内部参数，不建议修改 - -```yaml -# FATEFlow是否使用注册中心，使用注册中心的情况下，FATEFlow会注册FATEFlow Server地址以及发布的模型下载地址到注册中心以供在线系统FATEServing使用；同时也会从注册中心获取FATEServing地址 -use_registry: false -# 是否启用更高安全级别的序列化模式 -use_deserialize_safe_module: false -# fate on spark模式下是否启动依赖分发 -dependent_distribution: false -# 是否启动密码加密(数据库密码)，开启后配置encrypt_module和private_key才生效 -encrypt_password: false -# 加密包及加密函数(“#”号拼接) -encrypt_module: fate_arch.common.encrypt_utils#pwdecrypt -# 加密私钥 -private_key: -# 站点id，站点鉴权时需要配置 -party_id: -# 钩子模块配置 -hook_module: - # 客户端认证钩子 - client_authentication: fate_flow.hook.flow.client_authentication - # 站点端认证钩子 - site_authentication: fate_flow.hook.flow.site_authentication - # 权限认证钩子 - permission: fate_flow.hook.flow.permission -# 除了支持使用flow的钩子进行认证、鉴权，也支持使用第三方服务注册的认证和鉴权接口 -# 第三方认证、鉴权服务注册的服务名 -hook_server_name: -# 认证 -authentication: - # 客户端认证配置 - client: - # 客户端认证开关 - switch: false - http_app_key: - http_secret_key: - # 站点认证配置 - site: - # 认证开关 - switch: false -# 鉴权 -permission: - # 鉴权开关 - switch: false - # 组件鉴权开关 - component: false - # 数据集鉴权开关 - dataset: false -fateflow: - # 必须使用真实绑定的ip地址，避免因为多网卡/多IP引发的额外问题 - # you must set real ip address, 127.0.0.1 and 0.0.0.0 is not supported - host: 127.0.0.1 - http_port: 9380 - grpc_port: 9360 - # 高可用性时需要配置nginx地址 - nginx: - host: - http_port: - grpc_port: - # 实例id默认为{host}:{http_port}，可以通过random_instance_id配置生成随机id - random_instance_id: false - # 支持使用rollsite/nginx/fateflow作为多方任务协调通信代理 - # rollsite支持fate on eggroll的场景，仅支持grpc协议，支持P2P组网及星型组网模式 - # nginx支持所有引擎场景，支持http与grpc协议，默认为http，支持P2P组网及星型组网模式 - # fateflow支持所有引擎场景，支持http与grpc协议，默认为http，仅支持P2P组网模式，也即只支持互相配置对端fateflow地址 - # 格式(proxy: rollsite)表示使用rollsite并使用下方fate_one_eggroll配置大类中的rollsite配置；配置nginx表示使用下方fate_one_spark配置大类中的nginx配置 - # 也可以直接配置对端fateflow的地址，如下所示： - # proxy: - # name: fateflow - # host: xx - # http_port: xx - # grpc_port: xx - proxy: rollsite - # support default/http/grpc - protocol: default -database: - name: fate_flow - user: fate - passwd: fate - host: 127.0.0.1 - port: 3306 - max_connections: 100 - stale_timeout: 30 -# 注册中心地址及其身份认证参数 -zookeeper: - hosts: - - 127.0.0.1:2181 - use_acl: false - user: fate - password: fate -# engine services -default_engines: - computing: standalone - federation: standalone - storage: standalone -fate_on_standalone: - standalone: - cores_per_node: 20 - nodes: 1 -fate_on_eggroll: - clustermanager: - # eggroll nodemanager服务所在机器的CPU核数 - cores_per_node: 16 - # eggroll nodemanager服务的机器数量 - nodes: 1 - rollsite: - host: 127.0.0.1 - port: 9370 -fate_on_spark: - spark: - # default use SPARK_HOME environment variable - home: - cores_per_node: 20 - nodes: 2 - linkis_spark: - cores_per_node: 20 - nodes: 2 - host: 127.0.0.1 - port: 9001 - token_code: MLSS - python_path: /data/projects/fate/python - hive: - host: 127.0.0.1 - port: 10000 - auth_mechanism: - username: - password: - linkis_hive: - host: 127.0.0.1 - port: 9001 - hdfs: - name_node: hdfs://fate-cluster - # default / - path_prefix: - rabbitmq: - host: 192.168.0.4 - mng_port: 12345 - port: 5672 - user: fate - password: fate - # default conf/rabbitmq_route_table.yaml - route_table: - pulsar: - host: 192.168.0.5 - port: 6650 - mng_port: 8080 - cluster: standalone - # all parties should use a same tenant - tenant: fl-tenant - # message ttl in minutes - topic_ttl: 5 - # default conf/pulsar_route_table.yaml - route_table: - nginx: - host: 127.0.0.1 - http_port: 9300 - grpc_port: 9310 -# external services -fateboard: - host: 127.0.0.1 - port: 8080 - -# on API `/model/load` and `/model/load/do` -# automatic upload models to the model store if it exists locally but does not exist in the model storage -# or download models from the model store if it does not exist locally but exists in the model storage -# this config will not affect API `/model/store` or `/model/restore` -enable_model_store: false -# 模型导出(export model)操作默认的导出地址 -model_store_address: - # use mysql as the model store engine -# storage: mysql -# database: fate_model -# user: fate -# password: fate -# host: 127.0.0.1 -# port: 3306 - # other optional configs send to the engine -# max_connections: 10 -# stale_timeout: 10 - # use redis as the model store engine -# storage: redis -# host: 127.0.0.1 -# port: 6379 -# db: 0 -# password: - # the expiry time of keys, in seconds. defaults None (no expiry time) -# ex: - # use tencent cos as model store engine - storage: tencent_cos - Region: - SecretId: - SecretKey: - Bucket: - -# 不使用注册中心的情况下，需要配置FATE Serving Server的地址 -servings: - hosts: - - 127.0.0.1:8000 -fatemanager: - host: 127.0.0.1 - port: 8001 - federatedId: 0 - -``` - -## 3. FATE Flow配置 - -### 3.1 FATE Flow Server配置 - -- 路径：`${FATE_FLOW_BASE}/python/fate_flow/settings.py` -- 说明：高级配置，一般不需要做改动 -- 注意：配置文件中未被列举如下的配置项属于系统内部参数，不建议修改 - -```python -# FATE Flow Server用于多方FATE Flow Server通信的grpc server的线程池大小，不设置默认等于机器CPU核数 -GRPC_SERVER_MAX_WORKERS = None - -# Switch -# 上传数据接口默认从客户端获取数据，该值可以在接口调用时使用use_local_data配置自定义值 -UPLOAD_DATA_FROM_CLIENT = True -# 是否开启多方通信身份认证功能，需要配合FATE Cloud使用 -CHECK_NODES_IDENTITY = False -# 是否开启资源鉴权功能，需要配合FATE Cloud使用 -USE_AUTHENTICATION = False -# 默认授予的资源权限 -PRIVILEGE_COMMAND_WHITELIST = [] -``` - -### 3.2 FATE Flow 默认作业配置 - -- 路径：`${FATE_FLOW_BASE}/conf/job_default_config.yaml` -- 说明：高级配置，一般不需要做改动 -- 注意：配置文件中未被列举如下的配置项属于系统内部参数，不建议修改 -- 生效：使用flow server reload或者重启fate flow server - -```yaml -# component provider, relative path to get_fate_python_directory -default_component_provider_path: federatedml - -# resource -# 总资源超配百分比 -total_cores_overweight_percent: 1 # 1 means no overweight -total_memory_overweight_percent: 1 # 1 means no overweight -# 默认的每个作业的任务并行度，可以在提交作业配置时使用job_parameters:task_parallelism配置自定义值 -task_parallelism: 1 -# 默认的每个作业中每个任务使用的CPU核数，可以在提交作业配置时使用job_parameters:task_cores配置自定义值 -task_cores: 4 -# 暂时不支持内存资源的调度，该配置不生效 -task_memory: 0 # mb -# 一个作业最大允许申请的CPU核数占总资源数量的比例，如总资源为10，此值为0.5，则表示一个作业最多允许申请5个CPU，也即task_cores * task_parallelism <= 10 * 0.5 -max_cores_percent_per_job: 1 # 1 means total - -# scheduling -# 默认的作业执行超时时间，可以在提交作业配置时使用job_parameters:timeout配置自定义值 -job_timeout: 259200 # s -# 发送跨参与方调度命令或者状态时，通信的超时时间 -remote_request_timeout: 30000 # ms -# 发送跨参与方调度命令或者状态时，通信的重试次数 -federated_command_trys: 3 -end_status_job_scheduling_time_limit: 300000 # ms -end_status_job_scheduling_updates: 1 -# 默认自动重试次数, 可以在提交作业配置时使用job_parameters:auto_retries配置自定义值 -auto_retries: 0 -# 默认重试次数间隔 -auto_retry_delay: 1 #seconds -# 默认的多方状态收集方式，支持PULL和PUSH；也可在作业配置指定当前作业的收集模式 -federated_status_collect_type: PUSH - -# upload -upload_max_bytes: 104857600 # bytes - -#component output -output_data_summary_count_limit: 100 -``` - -## 4. FATE Board配置 - -- 路径：`${FATE_BOARD_BASE}/conf/application.properties` -- 说明：常用配置，一般部署时需要确定 -- 注意：配置文件中未被列举如下的配置项属于系统内部参数，不建议修改 - -```properties -# 服务监听端口 -server.port=8080 -# fateflow地址，指fateflow的http端口地址 -fateflow.url==http://127.0.0.1:9380 -# db地址，同上述全局配置service_conf.yaml里面的database配置 -fateboard.datasource.jdbc-url=jdbc:mysql://localhost:3306/fate_flow?characterEncoding=utf8&characterSetResults=utf8&autoReconnect=true&failOverReadOnly=false&serverTimezone=GMT%2B8 -# db配置，同上述全局配置service_conf.yaml里面的database配置 -fateboard.datasource.username= -# db配置，同上述全局配置service_conf.yaml里面的database配置 -fateboard.datasource.password= -server.tomcat.max-threads=1000 -server.tomcat.max-connections=20000 -spring.servlet.multipart.max-file-size=10MB -spring.servlet.multipart.max-request-size=100MB -# 管理员账号配置 -server.board.login.username=admin -server.board.login.password=admin -server.ssl.key-store=classpath: -server.ssl.key-store-password= -server.ssl.key-password= -server.ssl.key-alias= -# 当fateflo server开启api访问鉴权时，需要配置 -HTTP_APP_KEY= -HTTP_SECRET_KEY= -``` - -## 5. EggRoll - -### 5.1 系统配置 - -- 路径：`${EGGROLL_HOME}/conf/eggroll.properties` -- 说明：常用配置，一般部署时需要确定 -- 注意：配置文件中未被列举如下的配置项属于系统内部参数，不建议修改 - -```properties -[eggroll] -# core -# 连接MySQL配置，一般生产应用需要此配置 -eggroll.resourcemanager.clustermanager.jdbc.driver.class.name=com.mysql.cj.jdbc.Driver -# 连接MySQL配置，一般生产应用需要此配置 -eggroll.resourcemanager.clustermanager.jdbc.url=jdbc:mysql://localhost:3306/eggroll_meta?useSSL=false&serverTimezone=UTC&characterEncoding=utf8&allowPublicKeyRetrieval=true -# 连接MySQL账户，一般生产应用需要此配置 -eggroll.resourcemanager.clustermanager.jdbc.username= -# 连接MySQL密码，一般生产应用需要此配置 -eggroll.resourcemanager.clustermanager.jdbc.password= - -# 数据存储目录 -eggroll.data.dir=data/ -# 日志存储目录 -eggroll.logs.dir=logs/ -eggroll.resourcemanager.clustermanager.host=127.0.0.1 -eggroll.resourcemanager.clustermanager.port=4670 -eggroll.resourcemanager.nodemanager.port=4670 - -# python路径 -eggroll.resourcemanager.bootstrap.egg_pair.venv= -# pythonpath, 一般需要指定eggroll的python目录以及fate的python目录 -eggroll.resourcemanager.bootstrap.egg_pair.pythonpath=python - -# java路径 -eggroll.resourcemanager.bootstrap.egg_frame.javahome= -# java服务启动参数，无特别需要，无需配置 -eggroll.resourcemanager.bootstrap.egg_frame.jvm.options= -# 多方通信时，grpc连接保持时间 -eggroll.core.grpc.channel.keepalive.timeout.sec=20 - -# session -# 一个eggroll会话中，每个nodemanager启动的计算进程数量;若使用fate进行提交任务，则会被fate flow的默认参数所代替 -eggroll.session.processors.per.node=4 - -# rollsite -eggroll.rollsite.coordinator=webank -eggroll.rollsite.host=127.0.0.1 -eggroll.rollsite.port=9370 -eggroll.rollsite.party.id=10001 -eggroll.rollsite.route.table.path=conf/route_table.json - -eggroll.rollsite.push.max.retry=3 -eggroll.rollsite.push.long.retry=2 -eggroll.rollsite.push.batches.per.stream=10 -eggroll.rollsite.adapter.sendbuf.size=100000 -``` - -### 5.2 路由表配置 - -- 路径：`${EGGROLL_HOME}/conf/route_table.json` -- 说明：常用配置，一般部署时需要确定 - - 路由表主要分两个层级表示 - - 第一级表示站点，若找不到对应的目标站点配置，则使用**default** - - 第二级表示服务，若找不到对应的目标服务，则使用**default** - - 第二级，通常将**default**设为本方**rollsite**服务地址，将**fateflow**设为本方**fate flow server**服务的grpc地址 - -```json -{ - "route_table": - { - "10001": - { - "default":[ - { - "port": 9370, - "ip": "127.0.0.1" - } - ], - "fateflow":[ - { - "port": 9360, - "ip": "127.0.0.1" - } - ] - }, - "10002": - { - "default":[ - { - "port": 9470, - "ip": "127.0.0.1" - } - ] - } - }, - "permission": - { - "default_allow": true - } -} -``` \ No newline at end of file diff --git a/doc/document_navigation.md b/doc/document_navigation.md deleted file mode 100644 index ca8674a28..000000000 --- a/doc/document_navigation.md +++ /dev/null @@ -1,40 +0,0 @@ -# Document Navigation - -## 1. General Document Variables - -You will see the following `document variables` in all `FATE Flow` documentation, with the following meanings. - -- FATE_PROJECT_BASE: denotes the `FATE project` deployment directory, containing configuration, fate algorithm packages, fate clients and subsystems: `bin`, `conf`, `examples`, `fate`, `fateflow`, `fateboard`, `eggroll`, etc. -- FATE_BASE: The deployment directory of `FATE`, named `fate`, contains algorithm packages, clients: `federatedml`, `fate arch`, `fate client`, usually the path is `${FATE_PROJECT_BASE}/fate` -- FATE_FLOW_BASE: The deployment directory of `FATE Flow`, named `fateflow`, containing `fate flow server`, etc., usually the path is `${FATE_PROJECT_BASE}/fateflow` -- FATE_BOARD_BASE: the deployment directory of `FATE Board`, name `fateboard`, contains `fateboard`, usually the path is `${FATE_PROJECT_BASE}/fateboard` -- EGGROLL_HOME: the deployment directory for `EggRoll`, named `eggroll`, containing `rollsite`, `clustermanager`, `nodemanager`, etc., usually in `${FATE_PROJECT_BASE}/eggroll` - - Deploy the `FATE project` with reference to the main repository [FederatedAI/FATE](https://github.com/FederatedAI/FATE), the main directory structure is as follows - - ![](./images/fate_deploy_directory.png){: style="height:566px;width:212px"} - -- FATE_VERSION: The version number of `FATE`, e.g. 1.7.0 -- FATE_FLOW_VERSION: the version number of `FATE Flow`, e.g. 1.7.0 -- version: Generally in the deployment documentation, it means the version number of `FATE project`, such as `1.7.0`, `1.6.0`. -- version_tag: generally in the deployment documentation, indicates the `FATE project` version tag, such as `release`, `rc1`, `rc10` - -## 2. Glossary of terms - -`component_name`: the name of the component when the task is submitted, a task can have more than one of the same component, but the `component_name` is not the same, equivalent to an instance of the class - -`componet_module_name`: the class name of the component - -`model_alias`: similar to `component_name`, which is the name of the output model that the user can configure inside dsl - -Example. - -In the figure `dataio_0` is `component_name`, `DataIO` is `componet_module_name`, `dataio` is `model_alias` - -! [](https://user-images.githubusercontent.com/1758850/124451776-52ee4500-ddb8-11eb-94f2-d43d5174ca4d.png) - -## 3. Reading guide - -1. you can first read [overall design](. /fate_flow.zh.md) -2. Refer to the main repository [FATE](https://github.com/FederatedAI/FATE) for deployment, either standalone (installer, Docker, source compiler) or cluster (Ansible, Docker, Kuberneters) -3. You can refer to the directory in order of navigation diff --git a/doc/document_navigation.zh.md b/doc/document_navigation.zh.md deleted file mode 100644 index 160148421..000000000 --- a/doc/document_navigation.zh.md +++ /dev/null @@ -1,52 +0,0 @@ -# 文档导航 - -## 1. 通用文档变量 - -您会在所有`FATE Flow`的文档看到如下`文档变量`，其含义如下： - -- FATE_PROJECT_BASE：表示`FATE项目`部署目录，包含配置、fate算法包、fate客户端以及子系统: `bin`, `conf`, `examples`, `fate`, `fateflow`, `fateboard`, `eggroll`等 -- FATE_BASE：表示`FATE`的部署目录，名称`fate`，包含算法包、客户端: `federatedml`, `fate arch`, `fate client`, 通常路径为`${FATE_PROJECT_BASE}/fate` -- FATE_FLOW_BASE：表示`FATE Flow`的部署目录，名称`fateflow`，包含`fate flow server`等, 通常路径为`${FATE_PROJECT_BASE}/fateflow` -- FATE_BOARD_BASE：表示`FATE Board`的部署目录，名称`fateboard`，包含`fateboard`, 通常路径为`${FATE_PROJECT_BASE}/fateboard` -- EGGROLL_HOME：表示`EggRoll`的部署目录，名称`eggroll`，包含`rollsite`, `clustermanager`, `nodemanager`等, 通常路径为`${FATE_PROJECT_BASE}/eggroll` - - 参考`FATE项目`主仓库[FederatedAI/FATE](https://github.com/FederatedAI/FATE)部署`FATE项目`，主要目录结构如下： - - ![](./images/fate_deploy_directory.png){: style="height:566px;width:212px"} - -- FATE_VERSION：表示`FATE`的版本号，如1.7.0 -- FATE_FLOW_VERSION：表示`FATE Flow`的版本号，如1.7.0 -- version：一般在部署文档中，表示`FATE项目`版本号，如`1.7.0`, `1.6.0` -- version_tag：一般在部署文档中，表示`FATE项目`版本标签，如`release`, `rc1`, `rc10` - -## 2. 术语表 - -`party`, 站点，一般物理上指一个FATE单机或者FATE集群 - -`job`, 作业 - -`task`, 任务, 一个作业由多个任务构成 - -`component`, 组件，静态名称，提交作业时需要两个描述配置文件，分别描述该作业需要执行的组件列表、组件依赖关系、组件运行参数 - -`dsl`, 指用来描述作业中组件关系的语言, 可以描述组件列表以及组件依赖关系 - -`component_name`: 提交作业时组件的名称，一个作业可以有多个同样的组件的，但是 `component_name` 是不一样的，相当于类的实例, 一个`component_name`对应的组件会生成一个`task`运行 - -`componet_module_name`: 组件的类名 - -`model_alias`: 跟 `component_name` 类似，就是用户在 dsl 里面是可以配置输出的 model 名称的 - -示例： - -图中 `dataio_0` 是 `component_name`，`DataIO` 是 `componet_module_name`，`dataio` 是 `model_alias` - -![](https://user-images.githubusercontent.com/1758850/124451776-52ee4500-ddb8-11eb-94f2-d43d5174ca4d.png) - -`party status`, 指任务中每方的执行状态，`status`是由所有方的`party status`推断出，如所有`party status`为`success`，`status`才为success - -## 3. 阅读指引 - -1. 可以先阅读[整体设计](./fate_flow.zh.md) -2. 参考主仓库[FATE](https://github.com/FederatedAI/FATE)部署, 可选单机版(安装版, Docker, 源码编译)或集群版(Ansible, Docker, Kuberneters) -3. 可依据导航目录顺序进行参考 diff --git a/doc/faq.md b/doc/faq.md deleted file mode 100644 index 80fcce1ac..000000000 --- a/doc/faq.md +++ /dev/null @@ -1,102 +0,0 @@ -# FAQ - -## 1. Description - -## 2. Log descriptions - -In general, to troubleshoot a problem, the following logs are required. - -## v1.7+ - -- `${FATE_PROJECT_BASE}/fateflow/logs/$job_id/fate_flow_schedule.log`, this is the internal scheduling log of a certain task - -- `${FATE_PROJECT_BASE}/fateflow/logs/$job_id/*` These are all the execution logs of a certain task - -- `${FATE_PROJECT_BASE}/fateflow/logs/fate_flow/fate_flow_stat.log`, this is some logs that are not related to tasks - -- `${FATE_PROJECT_BASE}/fateflow/logs/fate_flow/fate_flow_schedule.log`, this is the overall scheduling log of all tasks - -- `${FATE_PROJECT_BASE}/fateflow/logs/fate_flow/fate_flow_detect.log`, which is the overall exception detection log for all tasks - -### v1.7- - -- `${FATE_PROJECT_BASE}/logs/$job_id/fate_flow_schedule.log`, this is the internal scheduling log for a particular task - -- `${FATE_PROJECT_BASE}/logs/$job_id/*` These are all the execution logs of a certain task - -- `${FATE_PROJECT_BASE}/logs/fate_flow/fate_flow_stat.log`, this is some logs that are not related to the task - -- `${FATE_PROJECT_BASE}/logs/fate_flow/fate_flow_schedule.log`, this is the overall scheduling log of all tasks - -- `${FATE_PROJECT_BASE}/logs/fate_flow/fate_flow_detect.log`, this is the overall exception detection log of all tasks - -## 3. Offline - -### upload failed - -- checking eggroll related services for exceptions. - -### submit job is stuck - -- check if both rollsite services have been killed - -### submit_job returns grpc exception - -- submit job link: guest fate_flow -> guest rollsite -> host rollsite -> host fate_flow -- check that each service in the above link is not hung, it must be ensured that each node is functioning properly. -- checking that the routing table is correctly configured. - -### dataio component exception: not enough values to unpack (expected 2, got 1) - -- the data separator does not match the separator in the configuration - -### Exception thrown at task runtime: "Count of data_instance is 0" - -- task has an intersection component and the intersection match rate is 0, need to check if the output data ids of guest and host can be matched. - -## 4. Serving - -### load model retcode returns 100, what are the possible reasons? - -- no fate-servings deployed - -- flow did not fetch the fate-servings address - -- flow reads the address of the fate-servings in priority order: - - 1. read from zk - - 2. if zk is not open, it will read from the fate-servings configuration file, the configuration path is - - - 1.5+: `${FATE_PROJECT_BASE}/conf/service_conf.yaml` - - - 1.5-: `${FATE_PROJECT_BASE}/arch/conf/server_conf.json` - -### load model retcode returns 123, what are the possible reasons? - -- Incorrect model information. -- This error code is thrown by fate-servings not finding the model. - -### bind model operation prompted "no service id"? - -- Customize the service_id in the bind configuration - -### Where is the configuration of servings? How do I configure it? - -- v1.5+ Configuration path: `${FATE_PROJECT_BASE}/conf/service_conf.yaml` - -```yaml -servings: - hosts: - - 127.0.0.1:8000 -``` - -- v1.5- Configuration path: `${FATE_PROJECT_BASE}/arch/conf/server_conf.json` - -```json -{ - "servers": { - "servings": ["127.0.0.1:8000"] - } -} -``` diff --git a/doc/faq.zh.md b/doc/faq.zh.md deleted file mode 100644 index 7f78cf7a8..000000000 --- a/doc/faq.zh.md +++ /dev/null @@ -1,102 +0,0 @@ -# 常见问题 - -## 1. 说明 - -## 2. 日志说明 - -一般来说，排查问题，需要如下几个日志： - -### v1.7+ - -- `${FATE_PROJECT_BASE}/fateflow/logs/$job_id/fate_flow_schedule.log`，这个是某个任务的内部调度日志 - -- `${FATE_PROJECT_BASE}/fateflow/logs/$job_id/*` 这些是某个任务的所有执行日志 - -- `${FATE_PROJECT_BASE}/fateflow/logs/fate_flow/fate_flow_stat.log`，这个是与任务无关的一些日志 - -- `${FATE_PROJECT_BASE}/fateflow/logs/fate_flow/fate_flow_schedule.log`，这个是所有任务的整体调度日志 - -- `${FATE_PROJECT_BASE}/fateflow/logs/fate_flow/fate_flow_detect.log`，这个是所有任务的整体异常探测日志 - -### v1.7- - -- `${FATE_PROJECT_BASE}/logs/$job_id/fate_flow_schedule.log`，这个是某个任务的内部调度日志 - -- `${FATE_PROJECT_BASE}/logs/$job_id/*` 这些是某个任务的所有执行日志 - -- `${FATE_PROJECT_BASE}/logs/fate_flow/fate_flow_stat.log`，这个是与任务无关的一些日志 - -- `${FATE_PROJECT_BASE}/logs/fate_flow/fate_flow_schedule.log`，这个是所有任务的整体调度日志 - -- `${FATE_PROJECT_BASE}/logs/fate_flow/fate_flow_detect.log`，这个是所有任务的整体异常探测日志 - -## 3. 离线部分 - -### upload失败 - -- 检查eggroll相关服务是否异常； - -### 提交任务(submit_job)卡住 - -- 检查双方rollsite服务是否被kill了 - -### 提交任务(submit_job)返回grpc异常 - -- 提交任务的链路： guest fate_flow -> guest rollsite -> host rollsite -> host fate_flow -- 检查上面的链路中的每个服务是否挂了，必须保证每个节点都正常运行； -- 检查路由表的配置是否正确； - -### dataio组件异常: not enough values to unpack (expected 2, got 1) - -- 数据的分隔符和配置中的分割符不一致 - -### 任务运行时抛出异常:"Count of data_instance is 0" - -- 任务中有交集组件并且交集匹配率为0，需要检查guest和host的输出数据id是否能匹配上； - -## 4. 在线部分 - -### 推模型(load)retcode返回100，可能的原因有哪些？ - -- 没有部署fate-servings - -- flow没有获取到fate-servings的地址 - -- flow读取fate-servings的地址的优先级排序: - - 1. 从zk读取 - - 2. 没有打开zk的话，会从fate的服务配置文件读取，配置路径在 - - - 1.5+: `${FATE_PROJECT_BASE}/conf/service_conf.yaml` - - - 1.5-: `${FATE_PROJECT_BASE}/arch/conf/server_conf.json` - -### 推模型(load)retcode返回123，可能原因有哪些？ - -- 模型信息有误； -- 此错误码是fate-servings没有找到模型而抛出的； - -### 绑定模型(bind)操作时提示"no service id"? - -- 在bind配置中自定义service_id - -### servings的配置在哪?怎么配？ - -- 1.5+ 配置路径: `${FATE_PROJECT_BASE}/conf/service_conf.yaml` - -```yaml -servings: - hosts: - - 127.0.0.1:8000 -``` - -- 1.5- 配置路径: `${FATE_PROJECT_BASE}/arch/conf/server_conf.json` - -```json -{ - "servers": { - "servings": ["127.0.0.1:8000"] - } -} -``` diff --git a/doc/fate_flow_authority_management.md b/doc/fate_flow_authority_management.md deleted file mode 100644 index 9996d6624..000000000 --- a/doc/fate_flow_authority_management.md +++ /dev/null @@ -1,152 +0,0 @@ -# Certification program - -## 1. Description - -- Authentication includes: client authentication and site authentication - -- Authentication configuration: ```$FATE_BASE/conf/service_conf.yaml```. - - ```yaml - ## Site authentication requires configuration of the party site id - party_id: - # Hook module, need to configure different hooks according to different scenarios - hook_module: - client_authentication: fate_flow.hook.flow.client_authentication - site_authentication: fate_flow.hook.flow.site_authentication - # Third-party authentication service name - hook_server_name: - authentication: - client: - # Client authentication switch - switch: false - http_app_key: - http_secret_key: - site: - # Site authentication switch - switch: false - ``` - -- Authentication method: Support flow's own authentication module authentication and third-party service authentication. The authentication hooks can be modified by hook_module, currently the following hooks are supported. - - client_authentication supports "fate_flow.hook.flow.client_authentication" and "fate_flow.hook.api.client_authentication", where the former is the client authentication method of flow. the former is the client authentication method of flow, the latter is the client authentication method of third-party services. - - site_authentication supports "fate_flow.hook.flow.site_authentication" and "fate_flow.hook.api.site_authentication", where the former is the site authentication method of flow and the latter is the third-party The former is the site authentication method of flow, and the latter is the third-party site authentication method. - - -## 2. client authentication - -### 2.1 flow authentication -#### 2.1.1 Configuration -```yaml -hook_module: - client_authentication: fate_flow.hook.flow.client_authentication -authentication: - client: - switch: true - http_app_key: "xxx" - http_secret_key: "xxx" -``` - - - -#### 2.2.2 Interface Authentication Method - -All client requests sent to Flow need to add the following header -``` - -`TIMESTAMP`: Unix timestamp in milliseconds, e.g. `1634890066095` means `2021-10-22 16:07:46 GMT+0800`, note that the difference between this time and the current time of the server cannot exceed 60 seconds - -`NONCE`: random string, can use UUID, such as `782d733e-330f-11ec-8be9-a0369fa972af` - -`APP_KEY`: must be consistent with `http_app_key` in the Flow configuration file - -`SIGNATURE`: signature generated based on `http_secret_key` and request parameters in the Flow configuration file - -``` -#### 2.2.3 Signature generation method - -- Combine the following elements in order - -`TIMESTAMP` - -`NONCE` - -`APP_KEY` - -request path + query parameters, if there are no query parameters then the final `? `, such as `/v1/job/submit` or `/v1/data/upload?table_name=dvisits_hetero_guest&namespace=experiment` - -If `Content-Type` is `application/json`, then it is the original JSON, i.e. the request body; if not, this item is filled with the empty string - -If `Content-Type` is `application/x-www-form-urlencoded` or `multipart/form-data`, all parameters need to be sorted alphabetically and `urlencode`, refer to RFC 3986 (i.e. except `a-zA-Z0-9- . _~`), note that the file does not participate in the signature; if not, this item is filled with the empty string - -- Concatenate all parameters with the newline character `\n` and encode them in `ASCII`. - -- Use the `HMAC-SHA1` algorithm to calculate the binary digest using the `http_secret_key` key in the Flow configuration file - -- Encode the binary digest using base64 - -#### 2.2.4 Example - -You can refer to [Fate SDK](https://github.com/FederatedAI/FATE/blob/master/python/fate_client/flow_sdk/client/base.py#L63) - -### 2.2 Third party service authentication -#### 2.2.1 Configuration -```yaml -hook_module: - client_authentication: fate_flow.hook.api.client_authentication -authentication: - client: - switch: true -hook_server_name: "xxx" -``` - -#### 2.2.2 Interface Authentication Method -- The third party service needs to register the client authentication interface with flow, refer to [Client Authentication Service Registration](./third_party_service_registry.md#321-client-authentication-client_authentication) -- If the authentication fails, flow will return the authentication failure directly to the client. - -## 3. Site Authentication - -### 3.1 flow authentication - -#### 3.1.1 Configuration -```yaml -party_id: 9999 -hook_module: - site_authentication: fate_flow.hook.flow.site_authentication -authentication: - client: - switch: true - http_app_key: "xxx" - http_secret_key: "xxx" -``` - -#### 3.1.2 Authentication scheme -- flow generates a pair of public and private keys when it starts, and needs to exchange public keys with each other with its partners. When sending a request, it uses the public key to generate a signature by RSA algorithm, and the requested site verifies the signature by its co-key. -- flow provides a key management cli as follows - -#### 3.1.3 Key Management -- Add the partner's public key - -{{snippet('cli/key.md', '### save')}} - -- Delete a partner's public key - -{{snippet('cli/key.md', '### delete')}} - - -- Query the co-key - -{{snippet('cli/key.md', '### query')}} - -### 3.2 Third-party service authentication -#### 3.2.1 Configuration -```yaml -hook_module: - site_authentication: fate_flow.hook.api.site_authentication -authentication: - site: - switch: true -hook_server_name: "xxx" -``` - -#### 3.2.2 Interface Authentication Method -- Third party services need to register the site authentication interface with flow, refer to [site authentication service registration](./third_party_service_registry.md#3222-site_authentication) -- If the authentication fails, flow will directly return the authentication failure to the initiator. \ No newline at end of file diff --git a/doc/fate_flow_authority_management.zh.md b/doc/fate_flow_authority_management.zh.md deleted file mode 100644 index 6a0010f13..000000000 --- a/doc/fate_flow_authority_management.zh.md +++ /dev/null @@ -1,153 +0,0 @@ -# 认证方案 - -## 1. 说明 - -- 认证包含：客户端认证和站点认证 - -- 认证配置: `$FATE_BASE/conf/service_conf.yaml`： - - ```yaml - # 站点鉴权时需要配置本方站点id - party_id: - # 钩子模块，需要根据不同场景配置不同的钩子 - hook_module: - client_authentication: fate_flow.hook.flow.client_authentication - site_authentication: fate_flow.hook.flow.site_authentication - # 第三方认证服务名 - hook_server_name: - authentication: - client: - # 客户端认证开关 - switch: false - http_app_key: - http_secret_key: - site: - # 站点认证开关 - switch: false - ``` - -- 认证方式：支持flow自带的认证模块认证和第三方服务认证。可通过hook_module修改认证钩子，当前支持如下钩子： - - client_authentication支持"fate_flow.hook.flow.client_authentication"和"fate_flow.hook.api.client_authentication", 其中前者是flow的客户端认证方式，后者是第三方服务客户端认证方式； - - site_authentication支持"fate_flow.hook.flow.site_authentication"和"fate_flow.hook.api.site_authentication",其中前者是flow的站点端认证方式，后者是第三方服务站点认证方式。 - - -## 2. 客户端认证 - -### 2.1 flow认证 -#### 2.1.1 配置 -`````yaml -hook_module: - client_authentication: fate_flow.hook.flow.client_authentication -authentication: - client: - switch: true - http_app_key: "xxx" - http_secret_key: "xxx" -````` - - - -#### 2.2.2 接口鉴权方式 - -则所有客户端发送到 Flow 的请求都需要增加以下 header - -`TIMESTAMP`：Unix timestamp，单位毫秒，如 `1634890066095` 表示 `2021-10-22 16:07:46 GMT+0800`，注意该时间与服务器当前时间的差距不能超过 60 秒 - -`NONCE`：随机字符串，可以使用 UUID，如 `782d733e-330f-11ec-8be9-a0369fa972af` - -`APP_KEY`：需与 Flow 配置文件中的 `http_app_key` 一致 - -`SIGNATURE`：基于 Flow 配置文件中的 `http_secret_key` 和请求参数生成的签名 - -#### 2.2.3 签名生成方法 - -- 按照顺序组合下列内容 - -`TIMESTAMP` - -`NONCE` - -`APP_KEY` - -请求路径+查询参数，如没有查询参数则不需要末尾的 `?`，如 `/v1/job/submit` 或 `/v1/data/upload?table_name=dvisits_hetero_guest&namespace=experiment` - -如果 `Content-Type` 为 `application/json`，则为原始 JSON，即 request body；如果不是，此项使用空字符串填充 - -如果 `Content-Type` 为 `application/x-www-form-urlencoded` 或 `multipart/form-data`，则需要把所有参数以字母顺序排序并 `urlencode`，转码方式参照 RFC 3986（即除 `a-zA-Z0-9-._~` 以外的字符都要转码），注意文件不参与签名；如果不是，此项使用空字符串填充 - -- 把所有参数用换行符 `\n` 连接然后以 `ASCII` 编码 - -- 使用 `HMAC-SHA1` 算法，以 Flow 配置文件中的 `http_secret_key` 为密钥，算出二进制摘要 - -- 使用 base64 编码二进制摘要 - -#### 2.2.4 示例 - -可以参考 [Fate SDK](https://github.com/FederatedAI/FATE/blob/master/python/fate_client/flow_sdk/client/base.py#L63) - - - - -### 2.2 第三方服务认证 -#### 2.2.1 配置 -```yaml -hook_module: - client_authentication: fate_flow.hook.api.client_authentication -authentication: - client: - switch: true -hook_server_name: "xxx" -``` - -#### 2.2.2 接口鉴权方式 -- 第三方服务需要向flow注册客户端认证接口，具体参考[客户端认证服务注册](./third_party_service_registry.zh.md#321-client_authentication) -- 若认证失败，flow会直接返回认证失败给客户端。 - -## 3. 站点认证 - -### 3.1 flow认证 - -#### 3.1.1 配置 -```yaml -party_id: 9999 -hook_module: - site_authentication: fate_flow.hook.flow.site_authentication -authentication: - client: - switch: true - http_app_key: "xxx" - http_secret_key: "xxx" -``` - -#### 3.1.2 认证方案 -- flow启动时会生成一对公钥和私钥，需要和合作方交换彼此的公钥，发送请求时通过RSA算法使用公钥生成签名，被请求站点通过其共钥验证签名。 -- flow提供密钥管理cli，如下 - -#### 3.1.3 密钥管理 -- 添加合作方公钥 - -{{snippet('cli/key.zh.md', '### save')}} - -- 删除合作方公钥 - -{{snippet('cli/key.zh.md', '### delete')}} - - -- 查询共钥 - -{{snippet('cli/key.zh.md', '### query')}} - -### 3.2 第三方服务认证 -#### 3.2.1 配置 -```yaml -hook_module: - site_authentication: fate_flow.hook.api.site_authentication -authentication: - site: - switch: true -hook_server_name: "xxx" -``` - -#### 3.2.2 接口鉴权方式 -- 第三方服务需要向flow注册站点认证接口，具体参考[站点认证服务注册](./third_party_service_registry.zh.md#3222-site_authentication) -- 若认证失败，flow会直接返回认证失败给发起方。 \ No newline at end of file diff --git a/doc/fate_flow_client.md b/doc/fate_flow_client.md deleted file mode 100644 index 992b2551d..000000000 --- a/doc/fate_flow_client.md +++ /dev/null @@ -1,164 +0,0 @@ -# FATE Flow Client - -## Description - -- Introduces how to install and use the `FATE Flow Client`, which is usually included in the `FATE Client`, which contains several clients of the `FATE Project`: `Pipeline`, `FATE Flow Client` and `FATE Test`. -- Introducing the command line provided by `FATE Flow Client`, all commands will have a common invocation entry, you can type `flow` in the command line to get all the command categories and their subcommands. - -```bash - [IN] - flow - - [OUT] - Usage: flow COMMAND [OPTIONS] - - Fate Flow Client - - Options. - -h, --help Show this message and exit. - - Commands: -h, --help - Component Component Operations - data Data Operations - init Flow CLI Init Command - Job Job Operations - model Model Operations - queue Queue Operations - table Table Operations - task Task Operations -``` - -For more information, please consult the following documentation or use the `flow --help` command. - -- All commands are described - -## Install FATE Client - -### Online installation - -FATE Client will be distributed to `pypi`, you can install the corresponding version directly using tools such as `pip`, e.g. - -```bash -pip install fate-client -``` - -or - -```bash -pip install atmosphere-client==${version} -``` - -### Installing on a FATE cluster - -Please install on a machine with version 1.5.1 and above of FATE. - -Installation command. - -```shell -cd $FATE_PROJECT_BASE/ -# Enter the virtual environment of FATE PYTHON -source bin/init_env.sh -# Execute the installation -cd fate/python/fate_client && python setup.py install -``` - -Once the installation is complete, type ``flow`` on the command line and enter, the installation will be considered successful if you get the following return. - -```shell -Usage: flow [OPTIONS] COMMAND [ARGS]... - - Fate Flow Client - -Options: - -h, --help Show this message and exit. - -Commands: - component Component Operations - data Data Operations - init Flow CLI Init Command - Job Job Operations - model Model Operations - queue Queue Operations - Table Table Operations - tag Tag Operations - task Task Operations -Task Operations - -## Initialization - -Before using the fate-client, you need to initialize it. It is recommended to use the configuration file of fate-client to initialize it. - -### Specify the fateflow service address - -```bash -### Specify the IP address and port of the fateflow service for initialization -flow init --ip 192.168.0.1 --port 9380 -``` - -### via the configuration file on the FATE cluster - -```shell -### Go to the FATE installation path, e.g. /data/projects/fate -cd $FATE_PROJECT_BASE/ -flow init -c conf/service_conf.yaml -``` - -The initialization is considered successful if you get the following return. - -```json -{ - "retcode": 0, - "retmsg": "Fate Flow CLI has been initialized successfully." -} -``` - -## Verify - -Mainly verify that the client can connect to the `FATE Flow Server`, e.g. try to query the current job status - -```bash -flow job query -``` - -Usually the `retcode` in the return is `0`. - -```json -{ - "data": [], - "retcode": 0, - "retmsg": "no job could be found" -} -``` - -If it returns something like the following, it means that the connection is not available, please check the network situation - -```json -{ - "retcode": 100, - "retmsg": "Connection refused. Please check if the fate flow service is started" -} -``` - -{{snippet('cli/data.md')}} - -{{snippet('cli/table.md')}} - -{{snippet('cli/job.md')}} - -{{snippet('cli/task.md')}} - -{{snippet('cli/tracking.md')}} - -{{snippet('cli/model.md')}} - -{{snippet('cli/checkpoint.md')}} - -{{snippet('cli/provider.md')}} - -{{snippet('cli/resource.md')}} - -{{snippet('cli/privilege.md')}} - -{{snippet('cli/tag.md')}} - -{{snippet('cli/server.md')}} diff --git a/doc/fate_flow_client.zh.md b/doc/fate_flow_client.zh.md deleted file mode 100644 index 2d240e101..000000000 --- a/doc/fate_flow_client.zh.md +++ /dev/null @@ -1,164 +0,0 @@ -# 命令行客户端 - -## 说明 - -- 介绍如何安装使用`FATE Flow Client`，其通常包含在`FATE Client`中，`FATE Client`包含了`FATE项目`多个客户端：`Pipeline`, `FATE Flow Client` 和 `FATE Test` -- 介绍`FATE Flow Client`提供的命令行，所有的命令将有一个共有调用入口，您可以在命令行中键入`flow`以获取所有的命令分类及其子命令。 - -```bash - [IN] - flow - - [OUT] - Usage: flow COMMAND [OPTIONS] - - Fate Flow Client - - Options： - -h, --help Show this message and exit. - - Commands： - component Component Operations - data Data Operations - init Flow CLI Init Command - job Job Operations - model Model Operations - queue Queue Operations - table Table Operations - task Task Operations -``` - -更多信息，请查阅如下文档或使用`flow --help`命令。 - -- 介绍所有命令使用说明 - -## 安装FATE Client - -### 在线安装 - -FATE Client会发布到`pypi`，可直接使用`pip`等工具安装对应版本，如 - -```bash -pip install fate-client -``` - -或者 - -```bash -pip install fate-client==${version} -``` - -### 在FATE集群上安装 - -请在装有1.5.1及其以上版本fate的机器中进行安装： - -安装命令： - -```shell -cd $FATE_PROJECT_BASE/ -# 进入FATE PYTHON的虚拟环境 -source bin/init_env.sh -# 执行安装 -cd fate/python/fate_client && python setup.py install -``` - -安装完成之后，在命令行键入`flow` 并回车，获得如下返回即视为安装成功： - -```shell -Usage: flow [OPTIONS] COMMAND [ARGS]... - - Fate Flow Client - -Options: - -h, --help Show this message and exit. - -Commands: - component Component Operations - data Data Operations - init Flow CLI Init Command - job Job Operations - model Model Operations - queue Queue Operations - table Table Operations - tag Tag Operations - task Task Operations -``` - -## 初始化 - -在使用fate-client之前需要对其进行初始化，推荐使用fate的配置文件进行初始化，初始化命令如下： - -### 指定fateflow服务地址 - -```bash -# 指定fateflow的IP地址和端口进行初始化 -flow init --ip 192.168.0.1 --port 9380 -``` - -### 通过FATE集群上的配置文件 - -```shell -# 进入FATE的安装路径，例如/data/projects/fate -cd $FATE_PROJECT_BASE/ -flow init -c conf/service_conf.yaml -``` - -获得如下返回视为初始化成功： - -```json -{ - "retcode": 0, - "retmsg": "Fate Flow CLI has been initialized successfully." -} -``` - -## 验证 - -主要验证客户端是否能连接上`FATE Flow Server`，如尝试查询当前的作业情况 - -```bash -flow job query -``` - -一般返回中的`retcode`为`0`即可 - -```json -{ - "data": [], - "retcode": 0, - "retmsg": "no job could be found" -} -``` - -如返回类似如下，则表明连接不上，请检查网络情况 - -```json -{ - "retcode": 100, - "retmsg": "Connection refused. Please check if the fate flow service is started" -} -``` - -{{snippet('cli/data.zh.md')}} - -{{snippet('cli/table.zh.md')}} - -{{snippet('cli/job.zh.md')}} - -{{snippet('cli/task.zh.md')}} - -{{snippet('cli/tracking.zh.md')}} - -{{snippet('cli/model.zh.md')}} - -{{snippet('cli/checkpoint.zh.md')}} - -{{snippet('cli/provider.zh.md')}} - -{{snippet('cli/resource.zh.md')}} - -{{snippet('cli/privilege.zh.md')}} - -{{snippet('cli/tag.zh.md')}} - -{{snippet('cli/server.zh.md')}} diff --git a/doc/fate_flow_component_registry.md b/doc/fate_flow_component_registry.md deleted file mode 100644 index 9b0376679..000000000 --- a/doc/fate_flow_component_registry.md +++ /dev/null @@ -1,19 +0,0 @@ -# Task Component Registry - -## 1. Description - -- After `FATE Flow` version 1.7, it started to support multiple versions of component packages at the same time, for example, you can put both `fate` algorithm component packages of `1.7.0` and `1.7.1` versions -- We refer to the provider of the algorithm component package as the `component provider`, and the `name` and `version` uniquely identify the `component provider`. -- When submitting a job, you can specify which component package to use for this job via `job dsl`, please refer to [componentprovider](./fate_flow_job_scheduling.md#35-Component-Providers) - -## 2. Default Component Provider - -Deploying a `FATE` cluster will include a default component provider, which is usually found in the `${FATE_PROJECT_BASE}/python/federatedml` directory - -## 3. current component provider - -{{snippet('cli/provider.md', '### list')}} - -## 4. new component provider - -{{snippet('cli/provider.md', '### register')}} \ No newline at end of file diff --git a/doc/fate_flow_component_registry.zh.md b/doc/fate_flow_component_registry.zh.md deleted file mode 100644 index 5460c5192..000000000 --- a/doc/fate_flow_component_registry.zh.md +++ /dev/null @@ -1,19 +0,0 @@ -# 任务组件注册中心 - -## 1. 说明 - -- `FATE Flow` 1.7版本后，开始支持多版本组件包同时存在，例如可以同时放入`1.7.0`和`1.7.1`版本的`fate`算法组件包 -- 我们将算法组件包的提供者称为`组件提供者`，`名称`和`版本`唯一确定`组件提供者` -- 在提交作业时，可通过`job dsl`指定本次作业使用哪个组件包，具体请参考[组件provider](./fate_flow_job_scheduling.zh.md#35-组件provider) - -## 2. 默认组件提供者 - -部署`FATE`集群将包含一个默认的组件提供者，其通常在 `${FATE_PROJECT_BASE}/python/federatedml` 目录下 - -## 3. 当前组件提供者 - -{{snippet('cli/provider.zh.md', '### list')}} - -## 4. 新组件提供者 - -{{snippet('cli/provider.zh.md', '### register')}} \ No newline at end of file diff --git a/doc/fate_flow_data_access.md b/doc/fate_flow_data_access.md deleted file mode 100644 index cf5c99613..000000000 --- a/doc/fate_flow_data_access.md +++ /dev/null @@ -1,136 +0,0 @@ -# Data Access - -## 1. Description - -- The storage tables of fate are identified by table name and namespace. - -- fate provides an upload component for users to upload data to a storage system supported by the fate compute engine. - -- If the user's data already exists in a storage system supported by fate, the storage information can be mapped to a fate storage table by table bind. - -- If the table bind's table storage type is not consistent with the current default engine, the reader component will automatically convert the storage type; - - - -## 2. data upload - -{{snippet('cli/data.md', '### upload')}} - -## 3. table binding - -{{snippet('cli/table.md', '### bind')}} - - -## 4. table information query - -{{snippet('cli/table.md', '### info')}} - -## 5. Delete table data - -{{snippet('cli/table.md', '### delete')}} - - - -## 6. Download data - -{{snippet('cli/data.md', '### download')}} - -## 7. disable data - -{{snippet('cli/table.md', '### disable')}} - -## 8. enable data - -{{snippet('cli/table.md', '### enable')}} - -## 9. delete disable data - -{{snippet('cli/table.md', '### disable-delete')}} - - -## 10. Writer - -{{snippet('cli/data.md', '### writer')}} - - -## 11. reader component - -**Brief description:** - -- The reader component is a data input component of fate; -- The reader component converts input data into data of the specified storage type; - -**Parameter configuration**: - -The input table of the reader is configured in the conf when submitting the job: - -```json -{ - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - } - } - } - } -} - -``` - -**Component Output** - -The output data storage engine of the component is determined by the configuration file conf/service_conf.yaml, with the following configuration items: - -```yaml -default_engines: - storage: eggroll -``` - -- The computing engine and storage engine have certain support dependencies on each other, the list of dependencies is as follows. - - | computing_engine | storage_engine | - | :--------------- | :---------------------------- | - | standalone | standalone | - | eggroll | eggroll | - | spark | hdfs(distributed), localfs(standalone) | - -- The reader component's input data storage type supports: eggroll, hdfs, localfs, mysql, path, etc; -- reader component output data type is determined by default_engines.storage configuration (except for path) - -## 12. api-reader - -**Brief description:** - -- The data input of api-reader component is id, and the data output is feature; -- request parameters can be user-defined, e.g. version number, back month, etc.. -- The component will request third-party services, and the third-party services need to implement upload, query, download interfaces and register with the fate flow, which can be referred to [api-reader related service registration](./third_party_service_registry.md#31-apireader) - -**Parameter configuration**: - -Configure the api-reader parameter in the conf when submitting the job: - -```json -{ - "role": { - "guest": { - "0": { "api_reader_0": { - "server_name": "xxx", - "parameters": { "version": "xxx"}, - "id_delimiter": ",", - "head": true - } - } - } - } -} -``` -Parameter meaning: -- server_name: the name of the service to be requested -- parameters: the parameters of the requested feature -- id_delimiter: the data separator to be returned -- head: whether the returned data contains a header or not diff --git a/doc/fate_flow_data_access.zh.md b/doc/fate_flow_data_access.zh.md deleted file mode 100644 index bf4e1da6f..000000000 --- a/doc/fate_flow_data_access.zh.md +++ /dev/null @@ -1,129 +0,0 @@ -# 数据接入 - -## 1. 说明 - -- fate的存储表是由table name和namespace标识。 - -- fate提供upload组件供用户上传数据至fate计算引擎所支持的存储系统内； - -- 若用户的数据已经存在于fate所支持的存储系统，可通过table bind方式将存储信息映射到fate存储表； - -- 若table bind的表存储类型与当前默认引擎不一致，reader组件会自动转化存储类型; - -## 2. 数据上传 - -{{snippet('cli/data.zh.md', '### upload')}} - -## 3. 表绑定 - -{{snippet('cli/table.zh.md', '### bind')}} - -## 4. 表信息查询 - -{{snippet('cli/table.zh.md', '### info')}} - -## 5. 删除表数据 - -{{snippet('cli/table.zh.md', '### delete')}} - -## 6. 数据下载 - -{{snippet('cli/data.zh.md', '### download')}} - -## 7. 将数据设置为“不可用”状态 - -{{snippet('cli/table.zh.md', '### disable')}} - -## 8. 将数据设置为“可用”状态 - -{{snippet('cli/table.zh.md', '### enable')}} - -## 9. 删除“不可用”数据 - -{{snippet('cli/table.zh.md', '### disable-delete')}} - -## 10. writer组件 - -{{snippet('cli/data.zh.md', '### writer')}} - -## 11. reader组件 - -**简要描述：** - -- reader组件为fate的数据输入组件; -- reader组件可将输入数据转化为指定存储类型数据; - -**参数配置**: - -submit job时的conf中配置reader的输入表: - -```json -{ - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - } - } - } - } -} - -``` - -**组件输出** - -组件的输出数据存储引擎是由配置决定，配置文件conf/service_conf.yaml,配置项为: - -```yaml -default_engines: - storage: eggroll -``` - -- 计算引擎和存储引擎之间具有一定的支持依赖关系，依赖列表如下： - - | computing_engine | storage_engine | - | :--------------- | :---------------------------- | - | standalone | standalone | - | eggroll | eggroll | - | spark | hdfs(分布式), localfs(单机版) | - -- reader组件输入数据的存储类型支持: eggroll、hdfs、localfs、mysql、path等; -- reader组件的输出数据类型由default_engines.storage配置决定(path除外) - -## 12. api-reader组件 - -**简要描述：** - -- api-reader组件的数据输入为id，数据输出为特征; -- 请求参数可以由用户自定义，如：版本号、回溯月份等； -- 组件会请求第三方服务，第三方服务需要实现upload、query、download接口并向fate flow注册，可参考[api-reader相关服务注册](./third_party_service_registry.zh.md#31-apireader) - -**参数配置**: - -submit job时的conf中配置api-reader参数: - -```json -{ - "role": { - "guest": { - "0": {"api_reader_0": { - "server_name": "xxx", - "parameters": {"version": "xxx"}, - "id_delimiter": ",", - "head": true - } - } - } - } -} -``` -参数含义： -- server_name: 需要请求的服务名 -- parameters: 需要请求的特征参数 -- id_delimiter：返回的数据分隔符 -- head: 返回的数据是否含有数据头 diff --git a/doc/fate_flow_http_api.md b/doc/fate_flow_http_api.md deleted file mode 100644 index ddbb7cbb2..000000000 --- a/doc/fate_flow_http_api.md +++ /dev/null @@ -1,17 +0,0 @@ -# REST API - -## 1. Description - -### 2. Error codes - -`400 Bad Request` request body has both json and form - -`401 Unauthorized` Missing one or more header(s) - -`400 Invalid TIMESTAMP` `TIMESTAMP` could not be parsed - -`425 TIMESTAMP is more than 60 seconds away from the server time` The `TIMESTAMP` in the header is more than 60 seconds away from the server time - -`401 Unknown APP_KEY` header in `APP_KEY` does not match `http_app_key` in the Flow configuration file - -`403 Forbidden` Signature verification failed diff --git a/doc/fate_flow_http_api.zh.md b/doc/fate_flow_http_api.zh.md deleted file mode 100644 index 513314a5c..000000000 --- a/doc/fate_flow_http_api.zh.md +++ /dev/null @@ -1,30 +0,0 @@ -# REST API - -## 1. 说明 - -## 2. 设计规范 - -### 2.1 HTTP Method - -- HTTP Method: 一律采用`POST` -- Content Type: application/json - -### 2.2 URL规则(现有) - -/一级/二级/N级/最后一级 - -- 一级：接口版本，如v1 -- 二级：主资源名称，如job -- N级：子资源名称，如list, 允许有多个N级 -- 最后一级：操作: create/update/query/get/delete - -### 2.3 URL规则(建议改进) - -/一级/二级/三级/四级/N级/最后一级 - -- 一级：系统名称: fate -- 三级：子系统名称: flow -- 二级：接口版本，如v1 -- 四级：主资源名称，如job -- N级：子资源名称，如list, 允许有多个N级 -- 最后一级：操作: create/update/query/get/delete diff --git a/doc/fate_flow_http_api_call_demo.md b/doc/fate_flow_http_api_call_demo.md deleted file mode 100644 index 889b63ec5..000000000 --- a/doc/fate_flow_http_api_call_demo.md +++ /dev/null @@ -1,640 +0,0 @@ -# REST API CLIENT - -## 1. Description -### Use python request fate flow api - -## 2. data upload/download - -```python -import json -import os - -import requests -from anaconda_project.internal.test.multipart import MultipartEncoder - -base_url = "http://127.0.0.1:9380/v1" - - -def upload(): - uri = "/data/upload" - file_name = "./data/breast_hetero_guest.csv" - with open(file_name, 'rb') as fp: - data = MultipartEncoder( - fields={'file': (os.path.basename(file_name), fp, 'application/octet-stream')} - ) - config_data = { - "file": file_name, - "id_delimiter": ",", - "head": 1, - "partition": 4, - "namespace": "experiment", - "table_name": "breast_hetero_guest" - } - - response = requests.post( - url=base_url + uri, - data=data, - params=json.dumps(config_data), - headers={'Content-Type': data.content_type} - ) - print(response.text) - - -def download(): - uri = "/data/download" - config_data = { - "output_path": "./download_breast_guest.csv", - "namespace": "experiment", - "table_name": "breast_hetero_guest" - } - response = requests.post(url=base_url + uri, json=config_data) - print(response.text) - - -def upload_history(): - uri = "/data/upload/history" - config_data = { - "limit": 5 - } - response = requests.post(url=base_url + uri, json=config_data) - print(response.text) - - -``` -## 3. table -```python -import requests - -base_url = "http://127.0.0.1:9380/v1" - - -def table_bind(): - uri = "/table/bind" - data = { - "head": 1, - "partition": 8, - "address": {"user": "fate", "passwd": "fate", "host": "127.0.0.1", "port": 3306, "db": "xxx", "name": "xxx"}, - "id_name": "id", - "feature_column": "y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12", - "engine": "MYSQL", - "id_delimiter": ",", - "namespace": "wzh", - "name": "wzh", - } - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def table_delete(): - uri = "/table/delete" - data = { - "table_name": "breast_hetero_guest", - "namespace": "experiment" - } - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def table_info(): - uri = "/table/table_info" - data = { - "table_name": "xxx", - "namespace": "xxx" - } - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def table_list(): - uri = "/table/list" - data = {"job_id": "202204221515021092240", "role": "guest", "party_id": "20001"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def tracking_source(): - uri = "/table/tracking/source" - data = {"table_name": "xxx", "namespace": "xxx"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def tracking_job(): - uri = "/table/tracking/job" - data = {"table_name": "xxx", "namespace": "xxx"} - res = requests.post(base_url + uri, json=data) - print(res.text) - -``` - -## 4. job - -```python - -import tarfile - -import requests - -base_url = "http:/127.0.0.1:9380/v1" - - -def submit(): - uri = "/job/submit" - data = { - "dsl": { - "components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "table" - ] - } - }, - "dataio_0": { - "module": "DataIO", - "input": { - "data": { - "data": [ - "reader_0.table" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "dataio" - ] - }, - "need_deploy": True - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "dataio_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ] - } - }, - "hetero_feature_binning_0": { - "module": "HeteroFeatureBinning", - "input": { - "data": { - "data": [ - "intersection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "hetero_feature_binning" - ] - } - }, - "hetero_feature_selection_0": { - "module": "HeteroFeatureSelection", - "input": { - "data": { - "data": [ - "hetero_feature_binning_0.train" - ] - }, - "isometric_model": [ - "hetero_feature_binning_0.hetero_feature_binning" - ] - }, - "output": { - "data": [ - "train" - ], - "model": [ - "selected" - ] - } - }, - "hetero_lr_0": { - "module": "HeteroLR", - "input": { - "data": { - "train_data": [ - "hetero_feature_selection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "hetero_lr" - ] - } - }, - "evaluation_0": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "hetero_lr_0.train" - ] - } - }, - "output": { - "data": [ - "evaluate" - ] - } - } - } - }, - "runtime_conf": { - "dsl_version": "2", - "initiator": { - "role": "guest", - "party_id": 20001 - }, - "role": { - "guest": [ - 20001 - ], - "host": [ - 10001 - ], - "arbiter": [ - 10001 - ] - }, - "job_parameters": { - "common": { - "task_parallelism": 2, - "computing_partitions": 8, - "task_cores": 4, - "auto_retries": 1 - } - }, - "component_parameters": { - "common": { - "intersection_0": { - "intersect_method": "raw", - "sync_intersect_ids": True, - "only_output_key": False - }, - "hetero_lr_0": { - "penalty": "L2", - "optimizer": "rmsprop", - "alpha": 0.01, - "max_iter": 3, - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - } - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - }, - "dataio_0": { - "with_label": True, - "label_name": "y", - "label_type": "int", - "output_format": "dense" - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - }, - "dataio_0": { - "with_label": False, - "output_format": "dense" - }, - "evaluation_0": { - "need_run": False - } - } - } - } - } - } - } - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def stop(): - uri = "/job/stop" - data = {"job_id": "202204251958539401540"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def rerun(): - uri = "/job/rerun" - data = {"job_id": "202204251958539401540"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def query(): - uri = "/job/query" - data = {"job_id": "202204251958539401540"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def list_job(): - uri = "/job/list/job" - data = {"limit": 1} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def update(): - uri = "/job/update" - data = {"job_id": "202204251958539401540", "role": "guest", "party_id": 20001, "notes": "this is a test"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def parameter_update(): - uri = "/job/parameter/update" - data = {"component_parameters": {"common": {"hetero_lr_0": {"max_iter": 10}}}, - "job_parameters": {"common": {"auto_retries": 2}}, "job_id": "202204251958539401540"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def config(): - uri = "/job/config" - data = {"job_id": "202204251958539401540"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def log_download(): - uri = "/job/log/download" - data = {"job_id": "202204251958539401540a"} - download_tar_file_name = "./test.tar.gz" - res = requests.post(base_url + uri, json=data) - with open(download_tar_file_name, "wb") as fw: - for chunk in res.iter_content(1024): - if chunk: - fw.write(chunk) - tar = tarfile.open(download_tar_file_name, "r:gz") - file_names = tar.getnames() - for file_name in file_names: - tar.extract(file_name) - tar.close() - - -def log_path(): - uri = "/job/log/path" - data = {"job_id": "202204251958539401540"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def task_query(): - uri = "/job/task/query" - data = {"job_id": "202204251958539401540", "role": "guest", "party_id": 20001} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def list_task(): - uri = "/job/list/task" - data = {"job_id": "202204251958539401540", "role": "guest", "party_id": 20001} - res = requests.post(base_url + uri, json=data) - print(res.text) - -def job_clean(): - uri = "/job/clean" - data = {"job_id": "202204251958539401540", "role": "guest", "party_id": 20001} - res = requests.post(base_url + uri, json=data) - print(res.text) - -def clean_queue(): - uri = "/job/clean/queue" - res = requests.post(base_url + uri) - print(res.text) - - -``` - -## 5. tracking -```python -import tarfile - -import requests - -base_url = "http://127.0.0.1:9380/v1" - - -def job_data_view(): - uri = "/tracking/job/data_view" - data = {"job_id": "202203311009181495690", "role": "guest", "party_id": 20001} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def component_metric_all(): - uri = "/tracking/component/metric/all" - data = {"job_id": "202203311009181495690", "role": "guest", "party_id": 20001, "component_name": "HeteroSecureBoost_0"} - res = requests.post(base_url + uri, json=data) - print(res.text) -# {"data":{"train":{"loss":{"data":[[0,0.6076415445876732],[1,0.5374539452565573],[2,0.4778598986135903],[3,0.42733599866560723],[4,0.38433409799127843]],"meta":{"Best":0.38433409799127843,"curve_name":"loss","metric_type":"LOSS","name":"train","unit_name":"iters"}}}},"retcode":0,"retmsg":"success"} - - -def component_metric(): - uri = "/tracking/component/metrics" - data = {"job_id": "202203311009181495690", "role": "guest", "party_id": 20001, "component_name": "Intersection_0"} - res = requests.post(base_url + uri, json=data) - print(res.text) - -def component_metric_data(): - uri = "/tracking/component/metric_data" - data = {"job_id": "202203311009181495690", - "role": "guest", - "party_id": 20001, - "component_name": "Intersection_0", - "metric_name": "intersection", - "metric_namespace": "train"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def component_parameters(): - uri = "/tracking/component/parameters" - data = {"job_id": "202203311009181495690", - "role": "guest", - "party_id": 20001, - "component_name": "Intersection_0"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def component_output_model(): - uri = "/tracking/component/output/model" - data = {"job_id": "202203311009181495690", - "role": "guest", - "party_id": 20001, - "component_name": "Intersection_0"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def component_output_data(): - uri = "/tracking/component/output/data" - data = {"job_id": "202203311009181495690", - "role": "guest", - "party_id": 20001, - "component_name": "Intersection_0"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def component_output_data_download(): - uri = "/tracking/component/output/data/download" - download_tar_file_name = "data.tar.gz" - data = {"job_id": "202203311009181495690", - "role": "guest", - "party_id": 20001, - "component_name": "Intersection_0"} - res = requests.get(base_url + uri, json=data) - print(res.text) - with open(download_tar_file_name, "wb") as fw: - for chunk in res.iter_content(1024): - if chunk: - fw.write(chunk) - tar = tarfile.open(download_tar_file_name, "r:gz") - file_names = tar.getnames() - for file_name in file_names: - tar.extract(file_name) - tar.close() - - -def component_output_data_table(): - uri = "/tracking/component/output/data/table" - data = {"job_id": "202203311009181495690", - "role": "guest", - "party_id": 20001, - "component_name": "Intersection_0a"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def component_component_summary_download(): - uri = "/tracking/component/summary/download" - data = {"job_id": "202203311009181495690", - "role": "guest", - "party_id": 20001, - "component_name": "Intersection_0"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def component_list(): - uri = "/tracking/component/list" - data = {"job_id": "202203311009181495690"} - res = requests.post(base_url + uri, json=data) - print(res.text) -component_list() -``` - -## 6. resource -```python -import requests - -base_url = "http://127.0.0.1:9380/v1" - - -def resource_query(): - uri = "/resource/query" - data = {"engine_name": "EGGROLL"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - - -def resource_return(): - uri = "/resource/return" - data = {"job_id": "202204261616175720130"} - res = requests.post(base_url + uri, json=data) - print(res.text) -resource_return() -``` - -## 7. permission -```python -import requests - -base_url = "http://127.0.0.1:9380/v1" - - -def grant_privilege(): - uri = "/permission/grant/privilege" - data = { - "src_role": "guest", - "src_party_id": "9999", - "privilege_role": "all", - "privilege_component": "all", - "privilege_command": "all" - } - res = requests.post(base_url + uri, json=data) - print(res.text) - -# grant_privilege() - -def delete_privilege(): - uri = "/permission/delete/privilege" - data = { - "src_role": "guest", - "src_party_id": "9999", - "privilege_role": "guest", - "privilege_component": "dataio", - "privilege_command": "create" - } - res = requests.post(base_url + uri, json=data) - print(res.text) - -# delete_privilege() - - -def query_privilege(): - uri = "/permission/query/privilege" - data = { - "src_role": "guest", - "src_party_id": "9999" - } - res = requests.post(base_url + uri, json=data) - print(res.text) - -query_privilege() - -``` - - diff --git a/doc/fate_flow_http_api_call_demo.zh.md b/doc/fate_flow_http_api_call_demo.zh.md deleted file mode 100644 index 9fe01f919..000000000 --- a/doc/fate_flow_http_api_call_demo.zh.md +++ /dev/null @@ -1,640 +0,0 @@ -# REST API 调用 - -## 1. 说明 -### 使用python请求fate flow 接口 - -## 2. 数据上传/下载 - -```python -import json -import os - -import requests -from anaconda_project.internal.test.multipart import MultipartEncoder - -base_url = "http://127.0.0.1:9380/v1" - - -def upload(): - uri = "/data/upload" - file_name = "./data/breast_hetero_guest.csv" - with open(file_name, 'rb') as fp: - data = MultipartEncoder( - fields={'file': (os.path.basename(file_name), fp, 'application/octet-stream')} - ) - config_data = { - "file": file_name, - "id_delimiter": ",", - "head": 1, - "partition": 4, - "namespace": "experiment", - "table_name": "breast_hetero_guest" - } - - response = requests.post( - url=base_url + uri, - data=data, - params=json.dumps(config_data), - headers={'Content-Type': data.content_type} - ) - print(response.text) - - -def download(): - uri = "/data/download" - config_data = { - "output_path": "./download_breast_guest.csv", - "namespace": "experiment", - "table_name": "breast_hetero_guest" - } - response = requests.post(url=base_url + uri, json=config_data) - print(response.text) - - -def upload_history(): - uri = "/data/upload/history" - config_data = { - "limit": 5 - } - response = requests.post(url=base_url + uri, json=config_data) - print(response.text) - - -``` -## 3. 数据表操作 -```python -import requests - -base_url = "http://127.0.0.1:9380/v1" - - -def table_bind(): - uri = "/table/bind" - data = { - "head": 1, - "partition": 8, - "address": {"user": "fate", "passwd": "fate", "host": "127.0.0.1", "port": 3306, "db": "xxx", "name": "xxx"}, - "id_name": "id", - "feature_column": "y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12", - "engine": "MYSQL", - "id_delimiter": ",", - "namespace": "wzh", - "name": "wzh", - } - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def table_delete(): - uri = "/table/delete" - data = { - "table_name": "breast_hetero_guest", - "namespace": "experiment" - } - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def table_info(): - uri = "/table/table_info" - data = { - "table_name": "xxx", - "namespace": "xxx" - } - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def table_list(): - uri = "/table/list" - data = {"job_id": "202204221515021092240", "role": "guest", "party_id": "20001"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def tracking_source(): - uri = "/table/tracking/source" - data = {"table_name": "xxx", "namespace": "xxx"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def tracking_job(): - uri = "/table/tracking/job" - data = {"table_name": "xxx", "namespace": "xxx"} - res = requests.post(base_url + uri, json=data) - print(res.text) - -``` - -## 4. 任务 - -```python - -import tarfile - -import requests - -base_url = "http:/127.0.0.1:9380/v1" - - -def submit(): - uri = "/job/submit" - data = { - "dsl": { - "components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "table" - ] - } - }, - "dataio_0": { - "module": "DataIO", - "input": { - "data": { - "data": [ - "reader_0.table" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "dataio" - ] - }, - "need_deploy": True - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "dataio_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ] - } - }, - "hetero_feature_binning_0": { - "module": "HeteroFeatureBinning", - "input": { - "data": { - "data": [ - "intersection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "hetero_feature_binning" - ] - } - }, - "hetero_feature_selection_0": { - "module": "HeteroFeatureSelection", - "input": { - "data": { - "data": [ - "hetero_feature_binning_0.train" - ] - }, - "isometric_model": [ - "hetero_feature_binning_0.hetero_feature_binning" - ] - }, - "output": { - "data": [ - "train" - ], - "model": [ - "selected" - ] - } - }, - "hetero_lr_0": { - "module": "HeteroLR", - "input": { - "data": { - "train_data": [ - "hetero_feature_selection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "hetero_lr" - ] - } - }, - "evaluation_0": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "hetero_lr_0.train" - ] - } - }, - "output": { - "data": [ - "evaluate" - ] - } - } - } - }, - "runtime_conf": { - "dsl_version": "2", - "initiator": { - "role": "guest", - "party_id": 20001 - }, - "role": { - "guest": [ - 20001 - ], - "host": [ - 10001 - ], - "arbiter": [ - 10001 - ] - }, - "job_parameters": { - "common": { - "task_parallelism": 2, - "computing_partitions": 8, - "task_cores": 4, - "auto_retries": 1 - } - }, - "component_parameters": { - "common": { - "intersection_0": { - "intersect_method": "raw", - "sync_intersect_ids": True, - "only_output_key": False - }, - "hetero_lr_0": { - "penalty": "L2", - "optimizer": "rmsprop", - "alpha": 0.01, - "max_iter": 3, - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - } - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - }, - "dataio_0": { - "with_label": True, - "label_name": "y", - "label_type": "int", - "output_format": "dense" - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - }, - "dataio_0": { - "with_label": False, - "output_format": "dense" - }, - "evaluation_0": { - "need_run": False - } - } - } - } - } - } - } - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def stop(): - uri = "/job/stop" - data = {"job_id": "202204251958539401540"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def rerun(): - uri = "/job/rerun" - data = {"job_id": "202204251958539401540"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def query(): - uri = "/job/query" - data = {"job_id": "202204251958539401540"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def list_job(): - uri = "/job/list/job" - data = {"limit": 1} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def update(): - uri = "/job/update" - data = {"job_id": "202204251958539401540", "role": "guest", "party_id": 20001, "notes": "this is a test"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def parameter_update(): - uri = "/job/parameter/update" - data = {"component_parameters": {"common": {"hetero_lr_0": {"max_iter": 10}}}, - "job_parameters": {"common": {"auto_retries": 2}}, "job_id": "202204251958539401540"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def config(): - uri = "/job/config" - data = {"job_id": "202204251958539401540"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def log_download(): - uri = "/job/log/download" - data = {"job_id": "202204251958539401540a"} - download_tar_file_name = "./test.tar.gz" - res = requests.post(base_url + uri, json=data) - with open(download_tar_file_name, "wb") as fw: - for chunk in res.iter_content(1024): - if chunk: - fw.write(chunk) - tar = tarfile.open(download_tar_file_name, "r:gz") - file_names = tar.getnames() - for file_name in file_names: - tar.extract(file_name) - tar.close() - - -def log_path(): - uri = "/job/log/path" - data = {"job_id": "202204251958539401540"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def task_query(): - uri = "/job/task/query" - data = {"job_id": "202204251958539401540", "role": "guest", "party_id": 20001} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def list_task(): - uri = "/job/list/task" - data = {"job_id": "202204251958539401540", "role": "guest", "party_id": 20001} - res = requests.post(base_url + uri, json=data) - print(res.text) - -def job_clean(): - uri = "/job/clean" - data = {"job_id": "202204251958539401540", "role": "guest", "party_id": 20001} - res = requests.post(base_url + uri, json=data) - print(res.text) - -def clean_queue(): - uri = "/job/clean/queue" - res = requests.post(base_url + uri) - print(res.text) - - -``` - -## 5. 指标 -```python -import tarfile - -import requests - -base_url = "http://127.0.0.1:9380/v1" - - -def job_data_view(): - uri = "/tracking/job/data_view" - data = {"job_id": "202203311009181495690", "role": "guest", "party_id": 20001} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def component_metric_all(): - uri = "/tracking/component/metric/all" - data = {"job_id": "202203311009181495690", "role": "guest", "party_id": 20001, "component_name": "HeteroSecureBoost_0"} - res = requests.post(base_url + uri, json=data) - print(res.text) -# {"data":{"train":{"loss":{"data":[[0,0.6076415445876732],[1,0.5374539452565573],[2,0.4778598986135903],[3,0.42733599866560723],[4,0.38433409799127843]],"meta":{"Best":0.38433409799127843,"curve_name":"loss","metric_type":"LOSS","name":"train","unit_name":"iters"}}}},"retcode":0,"retmsg":"success"} - - -def component_metric(): - uri = "/tracking/component/metrics" - data = {"job_id": "202203311009181495690", "role": "guest", "party_id": 20001, "component_name": "Intersection_0"} - res = requests.post(base_url + uri, json=data) - print(res.text) - -def component_metric_data(): - uri = "/tracking/component/metric_data" - data = {"job_id": "202203311009181495690", - "role": "guest", - "party_id": 20001, - "component_name": "Intersection_0", - "metric_name": "intersection", - "metric_namespace": "train"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def component_parameters(): - uri = "/tracking/component/parameters" - data = {"job_id": "202203311009181495690", - "role": "guest", - "party_id": 20001, - "component_name": "Intersection_0"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def component_output_model(): - uri = "/tracking/component/output/model" - data = {"job_id": "202203311009181495690", - "role": "guest", - "party_id": 20001, - "component_name": "Intersection_0"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def component_output_data(): - uri = "/tracking/component/output/data" - data = {"job_id": "202203311009181495690", - "role": "guest", - "party_id": 20001, - "component_name": "Intersection_0"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def component_output_data_download(): - uri = "/tracking/component/output/data/download" - download_tar_file_name = "data.tar.gz" - data = {"job_id": "202203311009181495690", - "role": "guest", - "party_id": 20001, - "component_name": "Intersection_0"} - res = requests.get(base_url + uri, json=data) - print(res.text) - with open(download_tar_file_name, "wb") as fw: - for chunk in res.iter_content(1024): - if chunk: - fw.write(chunk) - tar = tarfile.open(download_tar_file_name, "r:gz") - file_names = tar.getnames() - for file_name in file_names: - tar.extract(file_name) - tar.close() - - -def component_output_data_table(): - uri = "/tracking/component/output/data/table" - data = {"job_id": "202203311009181495690", - "role": "guest", - "party_id": 20001, - "component_name": "Intersection_0a"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def component_component_summary_download(): - uri = "/tracking/component/summary/download" - data = {"job_id": "202203311009181495690", - "role": "guest", - "party_id": 20001, - "component_name": "Intersection_0"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - -def component_list(): - uri = "/tracking/component/list" - data = {"job_id": "202203311009181495690"} - res = requests.post(base_url + uri, json=data) - print(res.text) -component_list() -``` - -## 6. 资源 -```python -import requests - -base_url = "http://127.0.0.1:9380/v1" - - -def resource_query(): - uri = "/resource/query" - data = {"engine_name": "EGGROLL"} - res = requests.post(base_url + uri, json=data) - print(res.text) - - - -def resource_return(): - uri = "/resource/return" - data = {"job_id": "202204261616175720130"} - res = requests.post(base_url + uri, json=data) - print(res.text) -resource_return() -``` - -## 7. 权限 -```python -import requests - -base_url = "http://127.0.0.1:9380/v1" - - -def grant_privilege(): - uri = "/permission/grant/privilege" - data = { - "src_role": "guest", - "src_party_id": "9999", - "privilege_role": "all", - "privilege_component": "all", - "privilege_command": "all" - } - res = requests.post(base_url + uri, json=data) - print(res.text) - -# grant_privilege() - -def delete_privilege(): - uri = "/permission/delete/privilege" - data = { - "src_role": "guest", - "src_party_id": "9999", - "privilege_role": "guest", - "privilege_component": "dataio", - "privilege_command": "create" - } - res = requests.post(base_url + uri, json=data) - print(res.text) - -# delete_privilege() - - -def query_privilege(): - uri = "/permission/query/privilege" - data = { - "src_role": "guest", - "src_party_id": "9999" - } - res = requests.post(base_url + uri, json=data) - print(res.text) - -query_privilege() - -``` - - diff --git a/doc/fate_flow_job_scheduling.md b/doc/fate_flow_job_scheduling.md deleted file mode 100644 index fa0b9fb86..000000000 --- a/doc/fate_flow_job_scheduling.md +++ /dev/null @@ -1,702 +0,0 @@ -# Multi-Party Job&Task Scheduling - -## 1. Description - -Mainly describes how to submit a federated learning job using `FATE Flow` and observe the use of - -## 2. Job submission - -- Build a federated learning job and submit it to the scheduling system for execution -- Two configuration files are required: job dsl and job conf -- job dsl configures the running components: list, input-output relationships -- job conf configures the component execution parameters, system operation parameters - -{{snippet('cli/job.md', '### submit')}} - -## 3. Job DSL configuration description - -The configuration file of DSL is in json format, in fact, the whole configuration file is a json object (dict). - -### 3.1 Component List - -**Description** The first level of this dict is `components`, which indicates the modules that will be used by this job. -**Example** - -```json -{ - "components" : { - ... - } -} -``` - -Each individual module is defined under "components", e.g. - -```json -"data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.train_data" - ] - } - }, - "output": { - "data": ["train"], - "model": ["model"] - } - } -``` - -All data needs to be fetched from the data store via the **Reader** module, note that this module only has the output `output` - -```json -"reader_0": { - "module": "Reader", - "output": { - "data": ["train"] - } -} -``` - -### 3.2 Modules - -**Description** Used to specify the components to be used, all optional module names refer to. -**Example** - -```json -"hetero_feature_binning_1": { - "module": "HeteroFeatureBinning", - ... -} -``` - -### 3.3 Inputs - -**Implications** Upstream inputs, divided into two input types, data and model. - -#### data input - -**Description** Upstream data input, divided into three input types. - - > 1. data: generally used in the data-transform module, feature_engineering module or - > evaluation module. - > 2. train_data: Generally used in homo_lr, hetero_lr and secure_boost - > modules. If the train_data field is present, then the task will be recognized as a fit task - > validate_data: If the train_data - > field is present, then the field is optional. If you choose to keep this field, the data pointed to will be used as the - > validation set - > 4. test_data: Used as prediction data, if provided, along with model input. - -#### model_input - -**Description** Upstream model input, divided into two input types. - 1. model: Used for model input of the same type of component. For example, hetero_binning_0 will fit the model, and then - hetero_binning_1 will use the output of hetero_binning_0 for predict or - transform. code example. - -```json - "hetero_feature_binning_1": { - "module": "HeteroFeatureBinning", - "input": { - "data": { - "data": [ - "data_transform_1.validate_data" - ] - }, - "model": [ - "hetero_feature_binning_0.fit_model" - ] - }, - "output": { - "data": ["validate_data" ], - "model": ["eval_model"] - } - } -``` - 2. isometric_model: Used to specify the model input of the inherited upstream component. For example, the upstream component of feature selection is - feature binning, it will use the information of feature binning as the feature - Code example. -```json - "hetero_feature_selection_0": { - "module": "HeteroFeatureSelection", - "input": { - "data": { - "data": [ - "hetero_feature_binning_0.train" - ] - }, - "isometric_model": [ - "hetero_feature_binning_0.output_model" - ] - }, - "output": { - "data": [ "train" ], - "model": ["output_model"] - } - } -``` - -### 3.4 Output - -**Description** Output, like input, is divided into data and model output - -#### data output - -**Description** Data output, divided into four output types. - -1. data: General module data output -2. train_data: only for Data Split -3. validate_data: Only for Data Split -4. test_data: Data Split only - -#### Model Output - -**Description** Model output, using model only - -### 3.5 Component Providers - -Since FATE-Flow version 1.7.0, the same FATE-Flow system supports loading multiple component providers, i.e. providers, which provide several components, and the source provider of the component can be configured when submitting a job -Since FATE-Flow version 1.9.0, the parameters of the provider need to be configured in job conf, as follows - -**Description** Specify the provider, support global specification and individual component specification; if not specified, the default provider: `fate@$FATE_VERSION` - -**Format** `provider_name@$provider_version` - -**Advanced** You can register a new provider through the component registration CLI, currently supported providers: fate and fate_sql, please refer to [FATE Flow Component Center](./fate_flow_component_registry.md) - -**Example** - -```json -{ - "dsl_version": "2", - "initiator": {}, - "role": {}, - "job_parameters": {}, - "component_parameters": {}, - "provider": { - "common": { - "hetero_feature_binning_0": "fate@1.8.0" - }, - "role": { - "guest": { - "0": { - "data_transform_0": "fate@1.9.0" - } - }, - "host": { - "0": { - "data_transform_0": "fate@1.9.0" - } - } - } - } -} -``` - -## 4. Job Conf Configuration Description - -Job Conf is used to set the information of each participant, the parameters of the job and the parameters of each component. The contents include the following. - -### 4.1 DSL Version - -**Description** Configure the version, the default is not 1, it is recommended to configure 2 -**Example** -```json -"dsl_version": "2" -``` - -### 4.2 Job participants - -#### initiating party - -**Description** The role and party_id of the assignment initiator. -**Example** -```json -"initiator": { - "role": "guest", - "party_id": 9999 -} -``` - -#### All participants - -**Description** Information about each participant. -**Description** In the role field, each element represents a role and the party_id that assumes that role. party_id for each role - The party_id of each role is in the form of a list, since a task may involve multiple parties in the same role. -**Example** - -```json -"role": { - "guest": [9999], - "host": [10000], - "arbiter": [10000] -} -``` - -### 4.3 System operation parameters - -**Description** - Configure the main system parameters for job runtime - -#### Parameter application scope policy setting - -**Apply to all participants, use the common scope identifier -**Apply to only one participant, use the role scope identifier, use (role:)party_index to locate the specified participant, direct - -```json -"common": { -} - -"role": { - "guest": { - "0": { - } - } -} -``` - -The parameters under common are applied to all participants, and the parameters under role-guest-0 configuration are applied to the participants under the subscript 0 of the guest role. -Note that the current version of the system operation parameters are not strictly tested for application to only one participant, so it is recommended to use common as a preference. - -#### Supported system parameters - -| Configuration | Default | Supported | Description | -| ----------------------------- | --------------------- | ------------------------------- | ------------------------------------------------------------------------------------------------- | -| job_type | train | train, predict | task_cores | -| task_cores | 4 | positive_integer | total_cpu_cores_applied_to_job | -| task_parallelism | 1 | positive_integer | task_parallelism | -| computing_partitions | number of cpu cores allocated to task | positive integer | number of partitions in the data table at computation time | -| eggroll_run | none | processors_per_node, etc. | eggroll computing engine related configuration parameters, generally do not need to be configured, from task_cores automatically calculated, if configured, task_cores parameters do not take effect | -| spark_run | none | num-executors, executor-cores, etc. | spark compute engine related configuration parameters, generally do not need to be configured, automatically calculated by task_cores, if configured, task_cores parameters do not take effect | -| rabbitmq_run | None | queue, exchange, etc. | Configuration parameters for rabbitmq to create queue, exchange, etc., which are generally not required and take the system defaults. -| pulsar_run | none | producer, consumer, etc. | The configuration parameters for pulsar to create producer and consumer. | -| federated_status_collect_type | PUSH | PUSH, PULL | Multi-party run status collection mode, PUSH means that each participant actively reports to the initiator, PULL means that the initiator periodically pulls from each participant. -| timeout | 259200 (3 days) | positive integer | task_timeout,unit_second | -| audo_retries | 3 | positive integer | maximum number of retries per task failure | -| model_id | \- | \- | The model id to be filled in for prediction tasks. -| model_version | \- | \- | Model version, required for prediction tasks - -1. there is a certain support dependency between the computation engine and the storage engine -2. developers can implement their own adapted engines, and configure the engines in runtime config - -#### reference configuration - -1. no need to pay attention to the compute engine, take the system default cpu allocation compute policy when the configuration - -```json -"job_parameters": { - "common": { - "job_type": "train", - "task_cores": 6, - "task_parallelism": 2, - "computing_partitions": 8, - "timeout": 36000 - } -} -``` - -2. use eggroll as the computing engine, take the configuration when specifying cpu and other parameters directly - -```json -"job_parameters": { - "common": { - "job_type": "train", - "eggroll_run": { - "eggroll.session.processors.per.node": 2 - }, - "task_parallelism": 2, - "computing_partitions": 8, - "timeout": 36000, - } -} -``` - -3. use spark as the computing engine, rabbitmq as the federation engine, take the configuration when specifying the cpu and other parameters directly - -```json -"job_parameters": { - "common": { - "job_type": "train", - "spark_run": { - "num-executors": 1, - "executor-cores": 2 - }, - "task_parallelism": 2, - "computing_partitions": 8, - "timeout": 36000, - "rabbitmq_run": { - "queue": { - "durable": true - }, - "connection": { - "heartbeat": 10000 - } - } - } -} -``` - -4. use spark as the computing engine and pulsar as the federation engine - -```json -"job_parameters": { - "common": { - "spark_run": { - "num-executors": 1, - "executor-cores": 2 - }, - } -} -``` -For more advanced resource-related configuration, please refer to [Resource Management](#4-Resource Management) - -### 4.3 Component operation parameters - -#### Parameter application scope policy setting - -- Apply to all participants, use common scope identifier -- Apply to only one participant, use the role scope identifier, use (role:)party_index to locate the specified participant, directly specified parameters have higher priority than common parameters - -```json -"commom": { -} - -"role": { - "guest": { - "0": { - } - } -} -``` - -where the parameters under the common configuration are applied to all participants, and the parameters under the role-guest-0 configuration indicate that they are applied to the participants under the subscript 0 of the guest role -Note that the current version of the component runtime parameter already supports two application scope policies - -#### Reference Configuration - -- For the `intersection_0` and `hetero_lr_0` components, the runtime parameters are placed under the common scope and are applied to all participants -- The operational parameters of `reader_0` and `data_transform_0` components are configured specific to each participant, because usually the input parameters are not consistent across participants, so usually these two components are set by participant -- The above component names are defined in the DSL configuration file - -```json -"component_parameters": { - "common": { - "intersection_0": { - "intersect_method": "raw", - "sync_intersect_ids": true, - "only_output_key": false - }, - "hetero_lr_0": { - "penalty": "L2", - "optimizer": "rmsprop", - "alpha": 0.01, - "max_iter": 3, - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - } - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": {"name": "breast_hetero_guest", "namespace": "experiment"} - }, - "data_transform_0":{ - "with_label": true, - "label_name": "y", - "label_type": "int", - "output_format": "dense" - } - } - }, - "host": { - "0": { - "reader_0": { - "table": {"name": "breast_hetero_host", "namespace": "experiment"} - }, - "data_transform_0":{ - "with_label": false, - "output_format": "dense" - } - } - } - } -} -``` - -## 5. Multi-Host Configuration - -Multi-Host task should list all host information under role - -**Example**: - -```json -"role": { - "guest": [ - 10000 - ], - "host": [ - 10000, 10001, 10002 - ], - "arbiter": [ - 10000 - ] -} -``` - -The different configurations for each host should be listed separately under their respective corresponding modules - -**Example**: - -```json -"component_parameters": { - "role": { - "host": { - "0": { - "reader_0": { - "table": - { - "name": "hetero_breast_host_0", - "namespace": "hetero_breast_host" - } - } - }, - "1": { - "reader_0": { - "table": - { - "name": "hetero_breast_host_1", - "namespace": "hetero_breast_host" - } - } - }, - "2": { - "reader_0": { - "table": - { - "name": "hetero_breast_host_2", - "namespace": "hetero_breast_host" - } - } - } - } - } -} -``` - -## 6. Predictive Task Configuration - -### 6.1 Description - -DSL V2 does not automatically generate prediction dsl for the training task. Users need to deploy the modules in the required model using `Flow Client` first. -For detailed command description, please refer to [fate_flow_client](./fate_flow_client.md) - -```bash -flow model deploy --model-id $model_id --model-version $model_version --cpn-list ... -``` - -Optionally, the user can add new modules to the prediction dsl, such as `Evaluation` - -### 6.2 Sample - -Training dsl. - -```json -"components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "data" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.data" - ] - } - }, - "output": { - "data":[ - "data" - ] - } - }, - "hetero_nn_0": { - "module": "HeteroNN", - "input": { - "data": { - "train_data": [ - "intersection_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - } -} -``` - -Prediction dsl: - -```json -"components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "data" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.data" - ] - } - }, - "output": { - "data":[ - "data" - ] - } - }, - "hetero_nn_0": { - "module": "HeteroNN", - "input": { - "data": { - "train_data": [ - "intersection_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "evaluation_0": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "hetero_nn_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ] - } - } -} -``` - -## 7. Job reruns - -In `1.5.0`, we started to support re-running a job, but only failed jobs are supported. -Version `1.7.0` supports rerunning of successful jobs, and you can specify which component to rerun from, the specified component and its downstream components will be rerun, but other components will not be rerun - -{{snippet('cli/job.md', '### rerun')}} - -## 8. Job parameter update - -In the actual production modeling process, it is necessary to constantly debug the component parameters and rerun, but not all components need to be adjusted and rerun at this time, so after `1.7.0` version support to modify a component parameter update, and with the `rerun` command on-demand rerun - -{{snippet('cli/job.md', '### parameter-update')}} - -## 9. Job scheduling policy - -- Queuing by commit time -- Currently, only FIFO policy is supported, i.e. the scheduler will only scan the first job each time, if the first job is successful in requesting resources, it will start and get out of the queue, if the request fails, it will wait for the next round of scheduling. - -## 10. dependency distribution - -**Brief description:** - -- Support for distributing fate and python dependencies from client nodes; -- The work node does not need to deploy fate; -- Only fate on spark supports distribution mode in current version; - -**Related parameters configuration**: - -conf/service_conf.yaml: - -```yaml -dependent_distribution: true -``` - -fate_flow/settings.py - -```python -FATE_FLOW_UPDATE_CHECK = False -``` - -**Description:** - -- dependent_distribution: dependent distribution switch;, off by default; when off, you need to deploy fate on each work node, and also fill in the configuration of spark in spark-env.sh to configure PYSPARK_DRIVER_PYTHON and PYSPARK_PYTHON. - -- FATE_FLOW_UPDATE_CHECK: Dependency check switch, turned off by default; it will automatically check if the fate code has changed every time a task is submitted; if it has changed, the fate code dependency will be re-uploaded; - -## 11. More commands - -Please refer to [Job CLI](./cli/job.md) and [Task CLI](./cli/task.md) \ No newline at end of file diff --git a/doc/fate_flow_job_scheduling.zh.md b/doc/fate_flow_job_scheduling.zh.md deleted file mode 100644 index f6e4292f6..000000000 --- a/doc/fate_flow_job_scheduling.zh.md +++ /dev/null @@ -1,702 +0,0 @@ -# 多方联合作业&任务调度 - -## 1. 说明 - -主要介绍如何使用`FATE Flow`提交一个联邦学习作业，并观察使用 - -## 2. 作业提交 - -- 构建一个联邦学习作业，并提交到调度系统执行 -- 需要两个配置文件：job dsl和job conf -- job dsl配置运行的组件：列表、输入输出关系 -- job conf配置组件执行参数、系统运行参数 - -{{snippet('cli/job.zh.md', '### submit')}} - -## 3. Job DSL配置说明 - -DSL 的配置文件采用 json 格式，实际上，整个配置文件就是一个 json 对象 （dict）。 - -### 3.1 组件列表 - -**描述** 在这个 dict 的第一级是 `components`，用来表示这个任务将会使用到的各个模块。 -**样例** - -```json -{ - "components" : { - ... - } -} -``` - -每个独立的模块定义在 "components" 之下，例如： - -```json -"data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.train_data" - ] - } - }, - "output": { - "data": ["train"], - "model": ["model"] - } - } -``` - -所有数据需要通过**Reader**模块从数据存储拿取数据，注意此模块仅有输出`output` - -```json -"reader_0": { - "module": "Reader", - "output": { - "data": ["train"] - } -} -``` - -### 3.2 模块 - -**描述** 用来指定使用的组件，所有可选module名称参考： -**样例** - -```json -"hetero_feature_binning_1": { - "module": "HeteroFeatureBinning", - ... -} -``` - -### 3.3 输入 - -**描述** 上游输入，分为两种输入类型，分别是数据和模型。 - -#### 数据输入 - -**描述** 上游数据输入，分为三种输入类型： - - > 1. data: 一般被用于 data-transform模块, feature_engineering 模块或者 - > evaluation 模块 - > 2. train_data: 一般被用于 homo_lr, hetero_lr 和 secure_boost - > 模块。如果出现了 train_data 字段，那么这个任务将会被识别为一个 fit 任务 - > 3. validate_data： 如果存在 train_data - > 字段，那么该字段是可选的。如果选择保留该字段，则指向的数据将会作为 - > validation set - > 4. test_data: 用作预测数据，如提供，需同时提供model输入。 - -#### 模型输入 - -**描述** 上游模型输入，分为两种输入类型： - 1. model: 用于同种类型组件的模型输入。例如，hetero_binning_0 会对模型进行 fit，然后 - hetero_binning_1 将会使用 hetero_binning_0 的输出用于 predict 或 - transform。代码示例： - -```json - "hetero_feature_binning_1": { - "module": "HeteroFeatureBinning", - "input": { - "data": { - "data": [ - "data_transform_1.validate_data" - ] - }, - "model": [ - "hetero_feature_binning_0.fit_model" - ] - }, - "output": { - "data": ["validate_data"], - "model": ["eval_model"] - } - } -``` - 2. isometric_model: 用于指定继承上游组件的模型输入。 例如，feature selection 的上游组件是 - feature binning，它将会用到 feature binning 的信息来作为 feature - importance。代码示例： -```json - "hetero_feature_selection_0": { - "module": "HeteroFeatureSelection", - "input": { - "data": { - "data": [ - "hetero_feature_binning_0.train" - ] - }, - "isometric_model": [ - "hetero_feature_binning_0.output_model" - ] - }, - "output": { - "data": ["train"], - "model": ["output_model"] - } - } -``` - -### 3.4 输出 - -**描述** 输出，与输入一样，分为数据和模型输出 - -#### 数据输出 - -**描述** 数据输出，分为四种输出类型： - -1. data: 常规模块数据输出 -2. train_data: 仅用于Data Split -3. validate_data: 仅用于Data Split -4. test_data： 仅用于Data Split - -#### 模型输出 - -**描述** 模型输出，仅使用model - -### 3.5 组件Provider - -FATE-Flow 1.7.0版本开始，同一个FATE-Flow系统支持加载多种且多版本的组件提供方，也即provider，provider提供了若干个组件，提交作业时可以配置组件的来源provider -FATE-Flow 1.9.0版本开始，provider的参数需在conf中配置，具体如下 - -**描述** 指定provider，支持全局指定以及单个组件指定；若不指定，默认 provider：`fate@$FATE_VERSION` - -**格式** `provider_name@$provider_version` - -**进阶** 可以通过组件注册CLI注册新的 provider，目前支持的 provider：fate 和 fate_sql，具体请参考[FATE Flow 组件中心](./fate_flow_component_registry.zh.md) - -**样例** - -```json -{ - "dsl_version": "2", - "initiator": {}, - "role": {}, - "job_parameters": {}, - "component_parameters": {}, - "provider": { - "common": { - "hetero_feature_binning_0": "fate@1.8.0" - }, - "role": { - "guest": { - "0": { - "data_transform_0": "fate@1.9.0" - } - }, - "host": { - "0": { - "data_transform_0": "fate@1.9.0" - } - } - } - } -} -``` - -## 4. Job Conf配置说明 - -Job Conf用于设置各个参与方的信息, 作业的参数及各个组件的参数。 内容包括如下： - -### 4.1 DSL版本 - -**描述** 配置版本，默认不配置为1，建议配置为2 -**样例** -```json -"dsl_version": "2" -``` - -### 4.2 作业参与方 - -#### 发起方 - -**描述** 任务发起方的role和party_id。 -**样例** -```json -"initiator": { - "role": "guest", - "party_id": 9999 -} -``` - -#### 所有参与方 - -**描述** 各参与方的信息。 -**说明** 在 role 字段中，每一个元素代表一种角色以及承担这个角色的 party_id。每个角色的 party_id - 以列表形式存在，因为一个任务可能涉及到多个 party 担任同一种角色。 -**样例** - -```json -"role": { - "guest": [9999], - "host": [10000], - "arbiter": [10000] -} -``` - -### 4.3 系统运行参数 - -**描述** - 配置作业运行时的主要系统参数 - -#### 参数应用范围策略设置 - -**应用于所有参与方，使用common范围标识符 -**仅应用于某参与方，使用role范围标识符，使用(role:)party_index定位被指定的参与方，直接指定的参数优先级高于common参数 - -```json -"common": { -} - -"role": { - "guest": { - "0": { - } - } -} -``` - -其中common下的参数应用于所有参与方，role-guest-0配置下的参数应用于guest角色0号下标的参与方 -注意，当前版本系统运行参数未对仅应用于某参与方做严格测试，建议使用优先选用common - -#### 支持的系统参数 - -| 配置项 | 默认值 | 支持值 | 说明 | -| ----------------------------- | --------------------- | ------------------------------- | ------------------------------------------------------------------------------------------------- | -| job_type | train | train, predict | 任务类型 | -| task_cores | 4 | 正整数 | 作业申请的总cpu核数 | -| task_parallelism | 1 | 正整数 | task并行度 | -| computing_partitions | task所分配到的cpu核数 | 正整数 | 计算时数据表的分区数 | -| eggroll_run | 无 | processors_per_node等 | eggroll计算引擎相关配置参数，一般无须配置，由task_cores自动计算得到，若配置则task_cores参数不生效 | -| spark_run | 无 | num-executors, executor-cores等 | spark计算引擎相关配置参数，一般无须配置，由task_cores自动计算得到，若配置则task_cores参数不生效 | -| rabbitmq_run | 无 | queue, exchange等 | rabbitmq创建queue、exchange的相关配置参数，一般无须配置，采取系统默认值 | -| pulsar_run | 无 | producer, consumer等 | pulsar创建producer和consumer时候的相关配置，一般无需配置。 | -| federated_status_collect_type | PUSH | PUSH, PULL | 多方运行状态收集模式，PUSH表示每个参与方主动上报到发起方，PULL表示发起方定期向各个参与方拉取 | -| timeout | 259200 (3天) | 正整数 | 任务超时时间,单位秒 | -| audo_retries | 3 | 正整数 | 每个任务失败自动重试最大次数 | -| model_id | \- | \- | 模型id，预测任务需要填入 | -| model_version | \- | \- | 模型version，预测任务需要填入 | - -1. 计算引擎和存储引擎之间具有一定的支持依赖关系 -2. 开发者可自行实现适配的引擎，并在runtime config配置引擎 - -#### 参考配置 - -1. 无须关注计算引擎，采取系统默认cpu分配计算策略时的配置 - -```json -"job_parameters": { - "common": { - "job_type": "train", - "task_cores": 6, - "task_parallelism": 2, - "computing_partitions": 8, - "timeout": 36000 - } -} -``` - -2. 使用eggroll作为computing engine，采取直接指定cpu等参数时的配置 - -```json -"job_parameters": { - "common": { - "job_type": "train", - "eggroll_run": { - "eggroll.session.processors.per.node": 2 - }, - "task_parallelism": 2, - "computing_partitions": 8, - "timeout": 36000, - } -} -``` - -3. 使用spark作为computing engine，rabbitmq作为federation engine,采取直接指定cpu等参数时的配置 - -```json -"job_parameters": { - "common": { - "job_type": "train", - "spark_run": { - "num-executors": 1, - "executor-cores": 2 - }, - "task_parallelism": 2, - "computing_partitions": 8, - "timeout": 36000, - "rabbitmq_run": { - "queue": { - "durable": true - }, - "connection": { - "heartbeat": 10000 - } - } - } -} -``` - -4. 使用spark作为computing engine，pulsar作为federation engine - -```json -"job_parameters": { - "common": { - "spark_run": { - "num-executors": 1, - "executor-cores": 2 - }, - } -} -``` -更多资源相关高级配置请参考[资源管理](#4-资源管理) - -### 4.3 组件运行参数 - -#### 参数应用范围策略设置 - -- 应用于所有参与方，使用common范围标识符 -- 仅应用于某参与方，使用role范围标识符，使用(role:)party_index定位被指定的参与方，直接指定的参数优先级高于common参数 - -```json -"commom": { -} - -"role": { - "guest": { - "0": { - } - } -} -``` - -其中common配置下的参数应用于所有参与方，role-guest-0配置下的参数表示应用于guest角色0号下标的参与方 -注意，当前版本组件运行参数已支持两种应用范围策略 - -#### 参考配置 - -- `intersection_0`与`hetero_lr_0`两个组件的运行参数，放在common范围下，应用于所有参与方 -- 对于`reader_0`与`data_transform_0`两个组件的运行参数，依据不同的参与方进行特定配置，这是因为通常不同参与方的输入参数并不一致，所有通常这两个组件一般按参与方设置 -- 上述组件名称是在DSL配置文件中定义 - -```json -"component_parameters": { - "common": { - "intersection_0": { - "intersect_method": "raw", - "sync_intersect_ids": true, - "only_output_key": false - }, - "hetero_lr_0": { - "penalty": "L2", - "optimizer": "rmsprop", - "alpha": 0.01, - "max_iter": 3, - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - } - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": {"name": "breast_hetero_guest", "namespace": "experiment"} - }, - "data_transform_0":{ - "with_label": true, - "label_name": "y", - "label_type": "int", - "output_format": "dense" - } - } - }, - "host": { - "0": { - "reader_0": { - "table": {"name": "breast_hetero_host", "namespace": "experiment"} - }, - "data_transform_0":{ - "with_label": false, - "output_format": "dense" - } - } - } - } -} -``` - -## 5. 多Host 配置 - -多Host任务应在role下列举所有host信息 - -**样例**: - -```json -"role": { - "guest": [ - 10000 - ], - "host": [ - 10000, 10001, 10002 - ], - "arbiter": [ - 10000 - ] -} -``` - -各host不同的配置应在各自对应模块下分别列举 - -**样例**: - -```json -"component_parameters": { - "role": { - "host": { - "0": { - "reader_0": { - "table": - { - "name": "hetero_breast_host_0", - "namespace": "hetero_breast_host" - } - } - }, - "1": { - "reader_0": { - "table": - { - "name": "hetero_breast_host_1", - "namespace": "hetero_breast_host" - } - } - }, - "2": { - "reader_0": { - "table": - { - "name": "hetero_breast_host_2", - "namespace": "hetero_breast_host" - } - } - } - } - } -} -``` - -## 6. 预测任务配置 - -### 6.1 说明 - -DSL V2不会自动为训练任务生成预测dsl。 用户需要首先使用`Flow Client`部署所需模型中模块。 -详细命令说明请参考[fate_flow_client](./fate_flow_client.zh.md) - -```bash -flow model deploy --model-id $model_id --model-version $model_version --cpn-list ... -``` - -可选地，用户可以在预测dsl中加入新模块，如`Evaluation` - -### 6.2 样例 - -训练 dsl： - -```json -"components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "data" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.data" - ] - } - }, - "output": { - "data":[ - "data" - ] - } - }, - "hetero_nn_0": { - "module": "HeteroNN", - "input": { - "data": { - "train_data": [ - "intersection_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - } -} -``` - -预测 dsl: - -```json -"components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "data" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.data" - ] - } - }, - "output": { - "data":[ - "data" - ] - } - }, - "hetero_nn_0": { - "module": "HeteroNN", - "input": { - "data": { - "train_data": [ - "intersection_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "evaluation_0": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "hetero_nn_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ] - } - } -} -``` - -## 7. 作业重跑 - -`1.5.0`版本, 开始支持重跑某个作业, 但是仅支持失败的作业 -`1.7.0`版本支持成功的作业重跑, 并且可以指定从哪个组件开始重跑, 被指定的组件及其下游组件会重跑, 但其他组件不会重跑 - -{{snippet('cli/job.zh.md', '### rerun')}} - -## 8. 作业参数更新 - -实际生产建模过程中, 需要进行不断调试修改组件参数且重跑, 但是此时并不是所有组件都需要调整并且重跑, 因此在`1.7.0`版本后支持修改某个组件的参数更新, 且配合`rerun`命令按需重跑 - -{{snippet('cli/job.zh.md', '### parameter-update')}} - -## 9. 作业调度策略 - -- 按提交时间先后入队 -- 目前仅支持FIFO策略，也即每次调度器仅会扫描第一个作业，若第一个作业申请资源成功则start且出队，若申请资源失败则等待下一轮调度 - -## 10. 依赖分发 - -**简要描述：** - -- 支持从client节点分发fate和python依赖; -- work节点不用部署fate; -- 当前版本只有fate on spark支持分发模式; - -**相关参数配置**: - -conf/service_conf.yaml: - -```yaml -dependent_distribution: true -``` - -fate_flow/settings.py - -```python -FATE_FLOW_UPDATE_CHECK = False -``` - -**说明：** - -- dependent_distribution: 依赖分发开关;，默认关闭;关闭时需要在每个work节点部署fate, 另外还需要在spark的配置spark-env.sh中填配置PYSPARK_DRIVER_PYTHON和PYSPARK_PYTHON； - -- FATE_FLOW_UPDATE_CHECK: 依赖校验开关, 默认关闭;打开后每次提交任务都会自动校验fate代码是否发生改变;若发生改变则会重新上传fate代码依赖; - -## 11. 更多命令 - -请参考[Job CLI](./cli/job.zh.md)和[Task CLI](./cli/task.zh.md) \ No newline at end of file diff --git a/doc/fate_flow_model_migration.md b/doc/fate_flow_model_migration.md deleted file mode 100644 index 217501688..000000000 --- a/doc/fate_flow_model_migration.md +++ /dev/null @@ -1,213 +0,0 @@ -# Inter-cluster Model Migration - -The model migration function makes it possible to copy the model file to a cluster with a different `party_id` and still have it available. - -1. the cluster of any of the model generation participants is redeployed and the `party_id` of the cluster is changed after the deployment, e.g. the source participant is `arbiter-10000#guest-9999#host-10000`, changed to `arbiter-10000#guest-99#host-10000` -2. Any one or more of the participants will copy the model file from the source cluster to the target cluster, which needs to be used in the target cluster - -Basics. -1. In the above two scenarios, the participant `party_id` of the model changes, such as `arbiter-10000#guest-9999#host-10000` -> `arbiter-10000#guest-99#host-10000`, or `arbiter-10000#guest -9999#host-10000` -> `arbiter-100#guest-99#host-100` -2. the model's participant `party_id` changes, so `model_id` and the model file involving `party_id` need to be changed -3. The overall process has three steps: copy and transfer the original model file, execute the model migration task on the original model file, and import the new model generated by the model migration task. -4. where *execute model migration task on the original model file* is actually a temporary copy of the original model file at the execution, and then modify `model_id` and the contents of the model file involving `party_id` according to the configuration, in order to adapt to the new participant `party_id`. -5. All the above steps need to be performed on all new participants, even if the `party_id` of one of the target participants has not changed. -6. the new participant cluster version needs to be greater than or equal to `1.5.1`. - -The migration process is as follows. - -## Transfer the model file - -Please package and transfer the model files (including the directory named by model id) generated by the machine where the source participant fate flow service is located to the machine where the target participant fate flow is located, and please transfer the model files to a fixed directory as follows. - -```bash -$FATE_PROJECT_BASE/model_local_cache -``` - -Instructions: -1. just transfer the folder, if you do the transfer by compressing and packing, please extract the model files to the directory where the model is located after the transfer. -2. Please transfer the model files one by one according to the source participants. - -## Preparation work before migration - -### Instructions - -1. refer to [fate flow client](. /fate_flow_client.zh.md) to install the client fate-client which supports model migration, only fate 1.5.1 and above are supported. - -## Execute the migration task - -### Description -1. Execute the migration task by replacing the source model file with `model_id`, `model_version` and the contents of the model involving `role` and `party_id` according to the migration task configuration file - -2. The cluster submitting the task must complete the above migration preparation - -### 1. Modify the configuration file - -Modify the configuration file of the migration task in the new participant (machine) according to the actual situation, as follows for the migration task example configuration file [migrate_model.json](https://github.com/FederatedAI/FATE-Flow/blob/main/examples/model /migrate_model.json) - -```json -{ - "job_parameters": { - "federated_mode": "SINGLE" - }, - "role": { - "guest": [9999], - "arbiter": [10000], - "host": [10000] - }, - "migrate_initiator": { - "role": "guest", - "party_id": 99 - }, - "migrate_role": { - "guest": [99], - "arbiter": [100], - "host": [100] - }, - "execute_party": { - "guest": [9999], - "arbiter": [10000], - "host": [10000] - }, - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "202006171904247702041", - "unify_model_version": "202901_0001" -} -``` - -Please save the above configuration content to a location in the server for modification. - -The following are explanatory notes for the parameters in this configuration. - -1. **`job_parameters`**: The `federated_mode` in this parameter has two optional parameters, which are `MULTIPLE` and `SINGLE`. If set to `SINGLE`, the migration job will be executed only in the party that submitted the migration job, then the job needs to be submitted in all new participants separately; if set to `MULTIPLE`, the job will be distributed to the participants specified in `execute_party` to execute the job, only the new The task will be distributed to the participant specified in `execute_party`, and only needs to be submitted in the new participant as `migrate_initiator`. -2. **`role`**: This parameter fills in the `role` of the participant that generated the original model and its corresponding `party_id` information. -3. **`migrate_initiator`**: This parameter is used to specify the task initiator information of the migrated model, and the initiator's `role` and `party_id` should be specified respectively. -4. **`migrate_role`**: This parameter is used to specify the `role` and `party_id` information of the migrated model. -5. **`execute_party`**: This parameter is used to specify the `role` and `party_id` information of the `party_id` that needs to execute the migration, which is the source cluster `party_id`. -6. **`model_id`**: This parameter is used to specify the `model_id` of the original model to be migrated. -7. **`model_version`**: This parameter is used to specify the `model_version` of the original model that needs to be migrated. -8. **`unify_model_version`**: This parameter is not required, it is used to specify the `model_version` of the new model. If this parameter is not provided, the new model will take the `job_id` of the migrated job as its new `model_version`. - -Examples of the above configuration files are. -1. the source model has `guest: 9999, host: 10000, arbiter: 10000,` migrate the model to have `guest: 99, host: 100, arbiter: 100` as participants, and `guest: 99` as the new initiator -2. `federated_mode: SINGLE` means that each migration task will be executed only in the cluster where the task is submitted, then the task needs to be submitted in 99 and 100 respectively. -3. for example, if the task is executed at 99, then `execute_party` is configured as `"guest": [9999]`. -4. For example, if you execute at 100, then `execute_party` is configured as `"arbiter": [10000], "host": [10000]` - - -## 2. Submit migration tasks (separate operations in all target clusters) - - -Migration tasks need to be committed using fate-client. A sample execution command is as follows. - -```bash -flow model migrate -c $FATE_FLOW_BASE/examples/model/migrate_model.json -``` - -## 3. Task execution results - -The following is the content of the configuration file for the actual migration task. - -```json -{ - "job_parameters": { - "federated_mode": "SINGLE" - }, - "role": { - "guest": [9999], - "host": [10000] - }, - "migrate_initiator": { - "role": "guest", - "party_id": 99 - }, - "migrate_role": { - "guest": [99], - "host": [100] - }, - "execute_party": { - "guest": [9999], - "host": [10000] - }, - "model_id": "guest-9999#host-10000#model", - "model_version": "202010291539339602784", - "unify_model_version": "fate_migration" -} -``` - -What this task achieves is to migrate the model with `model_id` of `guest-9999#host-10000#model` and `model_version` of `202010291539339602784` from a cluster with `party_id` of 9999 (guest) and 10000 (host) to a new model that fits the `party_id` of 99 (guest) and 100 (host) clusters - -The following is the result of a successful migration. - -```json -{ - "data": { - "detail": { - "guest": { - "9999": { - "retcode": 0, - "retmsg": "Migrating model successfully. the configuration of model has been modified automatically. new model id is: guest-99#host-100#model, Model files can be found at '/data/projects/fate/temp/fate_flow/guest#99#guest-99#host-100#model_fate_migration.zip'.zip. migration.zip'." - } - }, - "host": { - "10000": { - "retcode": 0, - "retmsg": "Migrating model successfully. The configuration of model has been modified automatically, Model files can be found at '/data/projects/fate/temp/fate_flow/host#100#guest-99#host-100#model_fate_migration.zip'.zip. migration.zip'." - } - } - }, - "guest": { - "9999": 0 - }, - "host": { - "10000": 0 - } - }, - "jobId": "202010292152299793981", - "retcode": 0, - "retmsg": "success" -} -``` - -After the task is successfully executed, a copy of the migrated model zip file is generated in each of the executor's machines, and the path to this file can be obtained in the returned results. For example, the path of the post-migration model file for 9999 (guest) is: `/data/projects/fate/temp/fate_flow/guest#99#guest-99#host-100#model_fate_migration.zip` and for 10000 (host) The model file path is: `/data/projects/fate/temp/fate_flow/host#100#guest-99#host-100#model_fate_migration.zip`. The new `model_id` can be obtained from the return as well as the `model_version`. - -## 4. Transferring files and importing (separate operation in all target clusters) - -After the migration task is successful, please manually transfer the newly generated model zip file to the fate flow machine of the target cluster. For example, the new model zip file generated by 9999 (guest) in point 3 needs to be transferred to the 99 (guest) machine. The zip file can be placed anywhere on the corresponding machine. Next, you need to configure the model import task, see [import_model.json](https://github.com/FederatedAI/FATE/blob/master/python/fate_flow/) for the configuration file examples/import_model.json) (this configuration file is included in the zip file, please modify it according to the actual situation, **do not use it directly**). - -The following is an example of the configuration file for importing the migrated model in guest (99). - -```json -{ - "role": "guest", - "party_id": 99, - "model_id": "guest-99#host-100#model", - "model_version": "202010292152299793981", - "file": "/data/projects/fate/python/temp/guest#99#guest-99#host-100#202010292152299793981.zip" -} -``` - -Please fill in the role `role`, the current party `party_id`, the new `model_id` and `model_version` of the migrated model, and the path to the zip file of the migrated model according to the actual situation. - -The following is a sample command to submit an imported model using fate-client. - -```bash -flow model import -c $FATE_FLOW_BASE/examples/model/import_model.json -``` - -The import is considered successful when it returns the following. - -```json -{ - "data": { - "job_id": "202208261102212849780", - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "foobar", - "party_id": "9999", - "role": "guest" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -The migration task is now complete and the user can submit the task with the new `model_id` and `model_version` to perform prediction tasks with the migrated model. diff --git a/doc/fate_flow_model_migration.zh.md b/doc/fate_flow_model_migration.zh.md deleted file mode 100644 index 4536e5c04..000000000 --- a/doc/fate_flow_model_migration.zh.md +++ /dev/null @@ -1,213 +0,0 @@ -# 集群间模型迁移 - -模型迁移功能使得模型文件复制拷贝到不同`party_id`的集群依然可用，以下两种场景需要做模型迁移： - -1. 模型生成参与方任何一方的集群, 重新部署且部署后集群的`party_id`变更, 例如源参与方为`arbiter-10000#guest-9999#host-10000`, 改为`arbiter-10000#guest-99#host-10000` -2. 其中任意一个或多个参与方将模型文件从源集群复制到目标集群，需要在目标集群使用 - -基本原理： -1. 上述两种场景下，模型的参与方`party_id`会发生改变，如`arbiter-10000#guest-9999#host-10000` -> `arbiter-10000#guest-99#host-10000`，或者`arbiter-10000#guest-9999#host-10000` -> `arbiter-100#guest-99#host-100` -2. 模型的参与方`party_id`发生改变，因此`model_id`以及模型文件里面涉及`party_id`需要改变 -3. 整体流程下来，有三个步骤：复制转移原有模型文件、对原有模型文件执行模型迁移任务、导入模型迁移任务生成的新模型 -4. 其中*原有模型文件执行模型迁移任务*其实就是在执行处临时复制一份原模型文件，然后按照配置，修改`model_id`及模型文件里面涉及`party_id`的内容，以适配新的参与方`party_id` -5. 上述步骤都需要在所有新的参与方执行，即使其中某个目标参与方的`party_id`没有改变，也需要执行 -6. 新的参与方集群版本需大于等于`1.5.1` - -迁移流程如下： - -## 转移模型文件 - -请将源参与方fate flow服务所在机器生成的模型文件（包括以model id为命名的目录）进行打包并转移到目标参与方fate flow所在机器中，请将模型文件转移至固定目录中： - -```bash -$FATE_PROJECT_BASE/model_local_cache -``` - -说明: -1. 文件夹转移即可，如果是通过压缩打包进行的转移，请在转移后将模型文件解压到模型所在目录中。 -2. 模型文件请按源目参与方一一对应转移 - -## 迁移前的准备工作 - -### 说明 - -1. 参考[fate flow client](./fate_flow_client.zh.md)安装支持模型迁移的客户端fate-client，只有fate 1.5.1及其以上版本支持 - -## 执行迁移任务 - -### 说明 -1. 执行迁移任务是将源模型文件根据迁移任务配置文件修改`model_id`、`model_version`以及模型内涉及`role`和`party_id`的内容进行替换 - -2. 提交任务的集群必须完成上述迁移准备 - -### 1. 修改配置文件 - -在新参与方（机器）中根据实际情况对迁移任务的配置文件进行修改，如下为迁移任务示例配置文件 [migrate_model.json](https://github.com/FederatedAI/FATE-Flow/blob/main/examples/model/migrate_model.json) - -```json -{ - "job_parameters": { - "federated_mode": "SINGLE" - }, - "role": { - "guest": [9999], - "arbiter": [10000], - "host": [10000] - }, - "migrate_initiator": { - "role": "guest", - "party_id": 99 - }, - "migrate_role": { - "guest": [99], - "arbiter": [100], - "host": [100] - }, - "execute_party": { - "guest": [9999], - "arbiter": [10000], - "host": [10000] - }, - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "202006171904247702041", - "unify_model_version": "20200901_0001" -} -``` - -请将上述配置内容保存到服务器中的某一位置进行修改。 - -以下为对该配置中的参数的解释说明： - -1. **`job_parameters`**：该参数中的`federated_mode`有两个可选参数，分别为`MULTIPLE` 及`SINGLE`。如果设置为`SINGLE`，则该迁移任务只会在提交迁移任务的本方执行，那么需要分别在所有新参与方提交任务;如果设置为`MULTIPLE`，则将任务分发到`execute_party`中指定的参与方执行任务，只需要在作为`migrate_initiator`的新参与方提交。 -2. **`role`**：该参数填写生成原始模型的参与方`role`及其对应的`party_id`信息。 -3. **`migrate_initiator`**：该参数用于指定迁移后的模型的任务发起方信息，分别需指定发起方的`role`与`party_id`。 -4. **`migrate_role`**：该参数用于指定迁移后的模型的参与方`role`及`party_id`信息。 -5. **`execute_party`**：该参数用于指定需要执行迁移的`role`及`party_id`信息, 该`party_id`为源集群`party_id`。 -6. **`model_id`**：该参数用于指定需要被迁移的原始模型的`model_id`。 -7. **`model_version`**：该参数用于指定需要被迁移的原始模型的`model_version`。 -8. **`unify_model_version`**：此参数为非必填参数，该参数用于指定新模型的`model_version`。若未提供该参数，新模型将以迁移任务的`job_id`作为其新`model_version`。 - -上述配置文件举例说明： -1. 源模型的参与方为`guest: 9999, host: 10000, arbiter: 10000,` 将模型迁移成参与方为`guest: 99, host: 100, arbiter: 100`, 且新发起方为`guest: 99` -2. `federated_mode: SINGLE` 表示每个迁移任务只在提交任务的集群执行任务，那么需要在99、100分别提交任务 -3. 例如在99执行，则`execute_party`配置为`"guest": [9999]` -4. 例如在100执行，则`execute_party`配置为`"arbiter": [10000], "host": [10000]` - - -## 2. 提交迁移任务(在所有目标集群分别操作) - - -迁移任务需使用fate-client进行提交，示例执行命令如下： - -```bash -flow model migrate -c $FATE_FLOW_BASE/examples/model/migrate_model.json -``` - -## 3. 任务执行结果 - -如下为实际迁移任务的配置文件内容： - -```json -{ - "job_parameters": { - "federated_mode": "SINGLE" - }, - "role": { - "guest": [9999], - "host": [10000] - }, - "migrate_initiator": { - "role": "guest", - "party_id": 99 - }, - "migrate_role": { - "guest": [99], - "host": [100] - }, - "execute_party": { - "guest": [9999], - "host": [10000] - }, - "model_id": "guest-9999#host-10000#model", - "model_version": "202010291539339602784", - "unify_model_version": "fate_migration" -} -``` - -该任务实现的是，将`party_id`为9999 (guest)，10000 (host)的集群生成的`model_id`为`guest-9999#host-10000#model`，`model_version`为`202010291539339602784`的模型修改迁移生成适配`party_id`为99 (guest)，100 (host)集群的新模型 - -如下为迁移成功的后得到的返回结果： - -```json -{ - "data": { - "detail": { - "guest": { - "9999": { - "retcode": 0, - "retmsg": "Migrating model successfully. The configuration of model has been modified automatically. New model id is: guest-99#host-100#model, model version is: fate_migration. Model files can be found at '/data/projects/fate/temp/fate_flow/guest#99#guest-99#host-100#model_fate_migration.zip'." - } - }, - "host": { - "10000": { - "retcode": 0, - "retmsg": "Migrating model successfully. The configuration of model has been modified automatically. New model id is: guest-99#host-100#model, model version is: fate_migration. Model files can be found at '/data/projects/fate/temp/fate_flow/host#100#guest-99#host-100#model_fate_migration.zip'." - } - } - }, - "guest": { - "9999": 0 - }, - "host": { - "10000": 0 - } - }, - "jobId": "202010292152299793981", - "retcode": 0, - "retmsg": "success" -} -``` - -任务成功执行后，执行方的机器中都会生成一份迁移后模型压缩文件，该文件路径可以在返回结果中得到。如上，9999 (guest)的迁移后模型文件路径为：`/data/projects/fate/temp/fate_flow/guest#99#guest-99#host-100#model_fate_migration.zip`，10000 (host)的迁移后模型文件路径为：`/data/projects/fate/temp/fate_flow/host#100#guest-99#host-100#model_fate_migration.zip`。新的`model_id`与`model_version`同样可以从返回中获得。 - -## 4. 转移文件并导入(在所有目标集群分别操作) - -迁移任务成功之后，请手动将新生成的模型压缩文件转移到目标集群的fate flow机器上。例如：第三点中9999 (guest)生成的新模型压缩文件需要被转移到99 (guest) 机器上。压缩文件可以放在对应机器上的任意位置，接下来需要配置模型的导入任务，配置文件请见[import_model.json](https://github.com/FederatedAI/FATE/blob/master/python/fate_flow/examples/import_model.json)（压缩文件内包含此配置文件，请根据实际情况修改，**切勿直接使用**）。 - -下面举例介绍在guest (99)中导入迁移后模型的配置文件： - -```json -{ - "role": "guest", - "party_id": 99, - "model_id": "guest-99#host-100#model", - "model_version": "202010292152299793981", - "file": "/data/projects/fate/python/temp/guest#99#guest-99#host-100#202010292152299793981.zip" -} -``` - -请根据实际情况对应填写角色`role`，当前本方`party_id`，迁移模型的新`model_id`及`model_version`，以及迁移模型的压缩文件所在路径。 - -如下为使用fate-client提交导入模型的示例命令： - -```bash -flow model import -c $FATE_FLOW_BASE/examples/model/import_model.json -``` - -得到如下返回视为导入成功： - -```json -{ - "data": { - "job_id": "202208261102212849780", - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "foobar", - "party_id": "9999", - "role": "guest" - }, - "retcode": 0, - "retmsg": "success" -} -``` - -迁移任务至此完成，用户可使用新的`model_id`及`model_version`进行任务提交，以利用迁移后的模型执行预测任务。 diff --git a/doc/fate_flow_model_registry.md b/doc/fate_flow_model_registry.md deleted file mode 100644 index ae921d9c2..000000000 --- a/doc/fate_flow_model_registry.md +++ /dev/null @@ -1,78 +0,0 @@ -# Federated Model Registry - -## 1. Description - -Models trained by FATE are automatically saved locally and recorded in the FATE-Flow database. models saved after each component run are called Pipeline models, and models saved at regular intervals while the component is running are called Checkpoint models. checkpoint models can also be used for retrying after a component run is unexpectedly interrupted The Checkpoint model can also be used for "breakpoints" when a component is retrying after an unexpected interruption. - -Checkpoint model support has been added since 1.7.0 and is not saved by default. To enable it, add the callback `ModelCheckpoint` to the DSL. - -### Local disk storage - -- Pipeline models are stored in `model_local_cache///variables/data//`. - -- Checkpoint models are stored in `model_local_cache///checkpoint//#`. - -### Remote storage engine - -Local disk is not reliable, so there is a risk of losing models. FATE-Flow supports exporting models to specified storage engines, importing from specified storage engines, and pushing models to engine storage when publishing models automatically. - -The storage engine supports Tencent Cloud Object Storage, MySQL and Redis, please refer to [Storage Engine Configuration](#4-storage-engine-configuration) - -## 2. Model - -{{snippet('cli/model.md', '## Model')}} - -## 3. Checkpoint - -{{snippet('cli/checkpoint.md', '## Checkpoint')}} - -## 4. Storage engine configuration - -### `enable_model_store` - -This option affects API `/model/load`. - -Automatic upload models to the model store if it exists locally but does not exist in the model storage, or download models from the model store if it does not exist locally but does not exist in the model storage. - -This option does not affect API `/model/store` or `/model/restore`. - -### `model_store_address` - -This config defines which storage engine to use. - -#### Tencent Cloud Object Storage - -```yaml -storage: tencent_cos -# get these configs from Tencent Cloud console -Region: -SecretId: -SecretKey: -Bucket: -``` - -#### MySQL - -```yaml -storage: mysql -database: fate_model -user: fate -password: fate -host: 127.0.0.1 -port: 3306 -# other optional configs send to the engine -max_connections: 10 -stale_timeout: 10 -``` - -#### Redis - -```yaml -storage: redis -host: 127.0.0.1 -port: 6379 -db: 0 -password: -# the expiry time of keys, in seconds. defaults None (no expiry time) -ex: -``` diff --git a/doc/fate_flow_model_registry.zh.md b/doc/fate_flow_model_registry.zh.md deleted file mode 100644 index b5b8c554b..000000000 --- a/doc/fate_flow_model_registry.zh.md +++ /dev/null @@ -1,203 +0,0 @@ -# 联合模型注册中心 - -## 1. 说明 - -由 FATE 训练的模型会自动保存到本地并记录在 FATE-Flow 的数据库中，每个组件运行完成后保存的模型称为 Pipeline 模型，在组件运行时定时保存的模型称为 Checkpoint 模型。Checkpoint 模型也可以用于组件运行意外中断后，重试时的“断点续传”。 - -Checkpoint 模型的支持自 1.7.0 加入，默认是不保存的，如需启用，则要向 DSL 中加入 callback `ModelCheckpoint`。 - -### 本地磁盘存储 - -- Pipeline 模型存储于 `model_local_cache///variables/data//`。 - -- Checkpoint 模型存储于 `model_local_cache///checkpoint//#`。 - -#### 目录结构 - -``` -tree model_local_cache/guest#9999#arbiter-10000#guest-9999#host-10000#model/202112181502241234200 - -model_local_cache/guest#9999#arbiter-10000#guest-9999#host-10000#model/202112181502241234200 -├── checkpoint -│   ├── data_transform_0 -│   ├── evaluation_0 -│   ├── hetero_linr_0 -│   │   ├── 0#step_name -│   │   │   ├── HeteroLinearRegressionMeta.json -│   │   │   ├── HeteroLinearRegressionMeta.pb -│   │   │   ├── HeteroLinearRegressionParam.json -│   │   │   ├── HeteroLinearRegressionParam.pb -│   │   │   └── database.yaml -│   │   ├── 1#step_name -│   │   │   ├── HeteroLinearRegressionMeta.json -│   │   │   ├── HeteroLinearRegressionMeta.pb -│   │   │   ├── HeteroLinearRegressionParam.json -│   │   │   ├── HeteroLinearRegressionParam.pb -│   │   │   └── database.yaml -│   │   ├── 2#step_name -│   │   │   ├── HeteroLinearRegressionMeta.json -│   │   │   ├── HeteroLinearRegressionMeta.pb -│   │   │   ├── HeteroLinearRegressionParam.json -│   │   │   ├── HeteroLinearRegressionParam.pb -│   │   │   └── database.yaml -│   │   ├── 3#step_name -│   │   │   ├── HeteroLinearRegressionMeta.json -│   │   │   ├── HeteroLinearRegressionMeta.pb -│   │   │   ├── HeteroLinearRegressionParam.json -│   │   │   ├── HeteroLinearRegressionParam.pb -│   │   │   └── database.yaml -│   │   └── 4#step_name -│   │   ├── HeteroLinearRegressionMeta.json -│   │   ├── HeteroLinearRegressionMeta.pb -│   │   ├── HeteroLinearRegressionParam.json -│   │   ├── HeteroLinearRegressionParam.pb -│   │   └── database.yaml -│   ├── hetero_linr_1 -│   ├── intersection_0 -│   └── reader_0 -├── define -│   ├── define_meta.yaml -│   ├── proto -│   │   └── pipeline.proto -│   └── proto_generated_python -│   ├── __pycache__ -│   │   └── pipeline_pb2.cpython-36.pyc -│   └── pipeline_pb2.py -├── run_parameters -│   ├── data_transform_0 -│   │   └── run_parameters.json -│   ├── hetero_linr_0 -│   │   └── run_parameters.json -│   ├── hetero_linr_1 -│   │   └── run_parameters.json -│   └── pipeline -│   └── run_parameters.json -└── variables - ├── data - │   ├── data_transform_0 - │   │   └── model - │   │   ├── DataTransformMeta - │   │   ├── DataTransformMeta.json - │   │   ├── DataTransformParam - │   │   └── DataTransformParam.json - │   ├── hetero_linr_0 - │   │   └── model - │   │   ├── HeteroLinearRegressionMeta - │   │   ├── HeteroLinearRegressionMeta.json - │   │   ├── HeteroLinearRegressionParam - │   │   └── HeteroLinearRegressionParam.json - │   ├── hetero_linr_1 - │   │   └── model - │   │   ├── HeteroLinearRegressionMeta - │   │   ├── HeteroLinearRegressionMeta.json - │   │   ├── HeteroLinearRegressionParam - │   │   └── HeteroLinearRegressionParam.json - │   └── pipeline - │   └── pipeline - │   ├── Pipeline - │   └── Pipeline.json - └── index - -32 directories, 47 files -``` - -**`checkpoint`** - -此目录存储组件运行过程中，每轮迭代产生的模型，不是所有组件都支持 checkpoint。 - -以 `checkpoint/hetero_linr_0/2#step_name` 为例： - -`hetero_linr_0` 是 `component_name`；`2` 是 `step_index`，即迭代次数；`step_name` 目前只做占位符，没有使用。 - -`HeteroLinearRegressionMeta.json`, `HeteroLinearRegressionMeta.pb`, `HeteroLinearRegressionParam.json`, `HeteroLinearRegressionParam.pb` 都是训练产生的数据，可以理解为模型文件。`database.yaml` 主要记录上述文件的 hash 以作校验，还存储有 `step_index`, `step_name`, `create_time`。 - -**`define`** - -该目录储存作业的基本信息，在作业初始化时创建。`pipeline` 不是一个组件，而是代表整个作业。 - -`define/proto/pipeline.proto` 和 `define/proto/pipeline_pb2.py` 目前没有使用。 - -`define/define_meta.yaml` 记录组件列表，包括 `component_name`, `componet_module_name`, `model_alias`。 - -**`run_parameters`** - -此目录存储组件的配置信息，也称为 DSL。 - -`run_parameters/pipeline/run_parameters.json` 为一个空的 object `{}`。 - -**`variables`** - -此目录存储组件运行结束后产生的模型，与最后一轮迭代产生的模型一致。 - -以 `variables/data/hetero_linr_0/model` 为例： - -`hetero_linr_0` 是 `component_name`；`model` 是 `model_alias`。 - -`HeteroLinearRegressionMeta`, `HeteroLinearRegressionMeta.json`, `HeteroLinearRegressionParam` `HeteroLinearRegressionParam.json` 与 `checkpoint` 目录下的文件格式完全一致，除了 `.pb` 文件去掉了扩展名。 - -`variables/data/pipeline/`存储作业的详细信息。 - -`variables/index/` 目前没有使用。 - -### 远端存储引擎 - -本地磁盘并不可靠，因此模型有丢失的风险，FATE-Flow 支持导出模型到指定存储引擎、从指定存储引擎导入以及自动发布模型时推送模型到引擎存储。 - -存储引擎支持腾讯云对象存储、MySQL 和 Redis, 具体请参考[存储引擎配置](#4-存储引擎配置) - -## 2. Model - -{{snippet('cli/model.zh.md', '## Model')}} - -## 3. Checkpoint - -{{snippet('cli/checkpoint.zh.md', '## Checkpoint')}} - -## 4. 存储引擎配置 - -### `enable_model_store` - -开启后，在调用 `/model/load` 时：如果模型文件在本地磁盘存在、但不在存储引擎中，则自动把模型文件上传至存储引擎；如果模型文件在存储引擎存在、但不在本地磁盘中，则自动把模型文件下载到本地磁盘。 - -此配置不影响 `/model/store` 和 `/model/restore`。 - -### `model_store_address` - -此配置定义使用的存储引擎。 - -#### 腾讯云对象存储 - -```yaml -storage: tencent_cos -# 请从腾讯云控制台获取下列配置 -Region: -SecretId: -SecretKey: -Bucket: -``` - -#### MySQL - -```yaml -storage: mysql -database: fate_model -user: fate -password: fate -host: 127.0.0.1 -port: 3306 -# 可选的数据库连接参数 -max_connections: 10 -stale_timeout: 10 -``` - -#### Redis - -```yaml -storage: redis -host: 127.0.0.1 -port: 6379 -db: 0 -password: -# key 的超时时间，单位秒。默认 None，没有超时时间。 -ex: -``` diff --git a/doc/fate_flow_monitoring.md b/doc/fate_flow_monitoring.md deleted file mode 100644 index 7bff4f7ad..000000000 --- a/doc/fate_flow_monitoring.md +++ /dev/null @@ -1,5 +0,0 @@ -# Real-Time Monitoring - -## 1. Description - -Mainly introduces `FATE Flow` to monitor job running status, Worker execution status, etc., in real time to ensure final consistency \ No newline at end of file diff --git a/doc/fate_flow_monitoring.zh.md b/doc/fate_flow_monitoring.zh.md deleted file mode 100644 index 7afb4e9f0..000000000 --- a/doc/fate_flow_monitoring.zh.md +++ /dev/null @@ -1,6 +0,0 @@ -# 作业实时监测 - -## 1. 说明 - -主要介绍`FATE Flow`对作业运行状态、Worker执行状态等，进行实时监测，以保证最终一致性 - diff --git a/doc/fate_flow_permission_management.md b/doc/fate_flow_permission_management.md deleted file mode 100644 index ce12d9156..000000000 --- a/doc/fate_flow_permission_management.md +++ /dev/null @@ -1,48 +0,0 @@ -## Multi-party cooperation rights management - -## 1. Description - -- fateflow permission authentication supports both flow's own authentication and third-party authentication - - -- Authentication configuration: ```$FATE_BASE/conf/service_conf.yaml```. - - ```yaml - hook_module: - permission: fate_flow.hook.flow.permission - hook_server_name: - permission: - switch: false - component: false - dataset: false - ``` - The permission hooks support both "fate_flow.hook.flow.permission" and "fate_flow.hook.api.permission". - -## 2. Permission authentication -### 2.1 flow permission authentication -#### 2.1.1 Authentication scheme -- The flow permission authentication scheme uses the casbin permission control framework and supports both component and dataset permissions. -- The configuration is as follows. -```yaml - hook_module: - permission: fate_flow.hook.flow.permission - permission: - switch: true - component: true - dataset: true -``` -#### 2.1.2 Authorization - -{{snippet('cli/privilege.md', '### grant')}} - -#### 2.1.3 Revoke privileges - -{{snippet('cli/privilege.md', '### delete')}} - -#### 2.1.4 Permission query - -{{snippet('cli/privilege.md', '### query')}} - -### 2.2 Third-party interface privilege authentication -- Third party services need to authenticate to the flow privilege interface, refer to [privilege authentication service registration](./third_party_service_registry.md#33-permission) -- If the authentication fails, flow will directly return the authentication failure to the partner. \ No newline at end of file diff --git a/doc/fate_flow_permission_management.zh.md b/doc/fate_flow_permission_management.zh.md deleted file mode 100644 index 1b550a0ae..000000000 --- a/doc/fate_flow_permission_management.zh.md +++ /dev/null @@ -1,48 +0,0 @@ -# 多方合作权限管理 - -## 1. 说明 - -- fateflow权限认证支持flow自身鉴权和第三方鉴权两种方式 - - -- 鉴权配置: `$FATE_BASE/conf/service_conf.yaml`： - - ```yaml - hook_module: - permission: fate_flow.hook.flow.permission - hook_server_name: - permission: - switch: false - component: false - dataset: false - ``` - 其中，权限钩子支持"fate_flow.hook.flow.permission"和"fate_flow.hook.api.permission"两种 - -## 2. 权限认证 -### 2.1 flow权限认证 -#### 2.1.1 认证方案 -- flow权限认证方案使用casbin权限控制框架，支持组件和数据集两种权限。 -- 配置如下： -```yaml - hook_module: - permission: fate_flow.hook.flow.permission - permission: - switch: true - component: true - dataset: true -``` -#### 2.1.2 授权 - -{{snippet('cli/privilege.zh.md', '### grant')}} - -#### 2.1.3 吊销权限 - -{{snippet('cli/privilege.zh.md', '### delete')}} - -#### 2.1.4 权限查询 - -{{snippet('cli/privilege.zh.md', '### query')}} - -### 2.2 第三方接口权限认证 -- 第三方服务需要向flow权限认证接口，具体参考[权限认证服务注册](./third_party_service_registry.zh.md#33-permission) -- 若认证失败，flow会直接返回认证失败给合作方。 \ No newline at end of file diff --git a/doc/fate_flow_resource_management.md b/doc/fate_flow_resource_management.md deleted file mode 100644 index 2a18fbf71..000000000 --- a/doc/fate_flow_resource_management.md +++ /dev/null @@ -1,102 +0,0 @@ -# Multi-Party Resource Coordination - -## 1. Description - -Resources refer to the basic engine resources, mainly CPU resources and memory resources of the compute engine, CPU resources and network resources of the transport engine, currently only the management of CPU resources of the compute engine is supported - -## 2. Total resource allocation - -- The current version does not automatically get the resource size of the base engine, so you configure it through the configuration file `$FATE_PROJECT_BASE/conf/service_conf.yaml`, that is, the resource size of the current engine allocated to the FATE cluster -- `FATE Flow Server` gets all the base engine information from the configuration file and registers it in the database table `t_engine_registry` when it starts. -- `FATE Flow Server` has been started and the resource configuration can be modified by restarting `FATE Flow Server` or by reloading the configuration using the command: `flow server reload`. -- `total_cores` = `nodes` * `cores_per_node` - -**Example** - -fate_on_standalone: is for executing a standalone engine on the same machine as `FATE Flow Server`, generally used for fast experiments, `nodes` is generally set to 1, `cores_per_node` is generally the number of CPU cores of the machine, also can be moderately over-provisioned - -```yaml -fate_on_standalone: - standalone: - cores_per_node: 20 - nodes: 1 -``` - -fate_on_eggroll: configured based on the actual deployment of `EggRoll` cluster, `nodes` denotes the number of `node manager` machines, `cores_per_node` denotes the average number of CPU cores per `node manager` machine - -```yaml -fate_on_eggroll: - clustermanager: - cores_per_node: 16 - nodes: 1 - rollsite: - host: 127.0.0.1 - port: 9370 -``` - -fate_on_spark: configured based on the resources allocated to the `FATE` cluster in the `Spark` cluster, `nodes` indicates the number of `Spark` nodes, `cores_per_node` indicates the average number of CPU cores per node allocated to the `FATE` cluster - -```yaml -fate_on_spark: - spark: - # default use SPARK_HOME environment variable - home: - cores_per_node: 20 - nodes: 2 -``` - -Note: Please make sure that the `Spark` cluster allocates the corresponding amount of resources to the `FATE` cluster, if the `Spark` cluster allocates less resources than the resources configured in `FATE` here, then it will be possible to submit the `FATE` job, but when `FATE Flow` submits the task to the `Spark` cluster, the task will not actually execute because the `Spark` cluster has insufficient resources. Insufficient resources, the task is not actually executed - -## 3. Job request resource configuration - -We generally use ``task_cores`'' and ``task_parallelism`' to configure job request resources, such as - -```json -{ -"job_parameters": { - "common": { - "job_type": "train", - "task_cores": 6, - "task_parallelism": 2, - "computing_partitions": 8, - "timeout": 36000 - } - } -} -``` - -The total resources requested by the job are `task_cores` * `task_parallelism`. When creating a job, `FATE Flow` will distribute the job to each `party` based on the above configuration, running role, and the engine used by the party (via `$FATE_PROJECT_BASE/conf/service_conf .yaml#default_engines`), the actual parameters will be calculated as follows - -## 4. The process of calculating the actual parameter adaptation for resource requests - -- Calculate `request_task_cores`: - - guest, host. - - `request_task_cores` = `task_cores` - - arbiter, considering that the actual operation consumes very few resources: `request_task_cores - - `request_task_cores` = 1 - -- Further calculate `task_cores_per_node`. - - `task_cores_per_node"` = max(1, `request_task_cores` / `task_nodes`) - - - If `eggroll_run` or `spark_run` configuration resource is used in the above `job_parameters`, then the `task_cores` configuration is invalid; calculate `task_cores_per_node`. - - `task_cores_per_node"` = eggroll_run["eggroll.session.processors.per.node"] - - `task_cores_per_node"` = spark_run["executor-cores"] - -- The parameter to convert to the adaptation engine (which will be presented to the compute engine for recognition when running the task). - - fate_on_standalone/fate_on_eggroll: - - eggroll_run["eggroll.session.processors.per.node"] = `task_cores_per_node` - - fate_on_spark: - - spark_run["num-executors"] = `task_nodes` - - spark_run["executor-cores"] = `task_cores_per_node` - -- The final calculation can be seen in the job's `job_runtime_conf_on_party.json`, typically in `$FATE_PROJECT_BASE/jobs/$job_id/$role/$party_id/job_runtime_on_party_conf.json ` - -## 5. Resource Scheduling Policy -- `total_cores` see [total_resource_allocation](#2-total-resource-allocation) -- `apply_cores` see [job_request_resource_configuration](#3-job-request-resource-configuration), `apply_cores` = `task_nodes` * `task_cores_per_node` * `task_parallelism` -- If all participants apply for resources successfully (total_cores - apply_cores) > 0, then the job applies for resources successfully -- If not all participants apply for resources successfully, then send a resource rollback command to the participants who have applied successfully, and the job fails to apply for resources - -## 6. Related commands - -{{snippet('cli/resource.md', header=False)}} diff --git a/doc/fate_flow_resource_management.zh.md b/doc/fate_flow_resource_management.zh.md deleted file mode 100644 index 4ab37f315..000000000 --- a/doc/fate_flow_resource_management.zh.md +++ /dev/null @@ -1,103 +0,0 @@ -# 多方资源协调 - -## 1. 说明 - -资源指基础引擎资源，主要指计算引擎的CPU资源和内存资源，传输引擎的CPU资源和网络资源，目前仅支持计算引擎CPU资源的管理 - -## 2. 总资源配置 - -- 当前版本未实现自动获取基础引擎的资源大小，因此你通过配置文件`$FATE_PROJECT_BASE/conf/service_conf.yaml`进行配置，也即当前引擎分配给FATE集群的资源大小 -- `FATE Flow Server`启动时从配置文件获取所有基础引擎信息并注册到数据库表`t_engine_registry` -- `FATE Flow Server`已经启动，修改资源配置，可重启`FATE Flow Server`，也可使用命令：`flow server reload`，重新加载配置 -- `total_cores` = `nodes` * `cores_per_node` - -**样例** - -fate_on_standalone：是为执行在`FATE Flow Server`同台机器的单机引擎，一般用于快速实验，`nodes`一般设置为1，`cores_per_node`一般为机器CPU核数，也可适量超配 - -```yaml -fate_on_standalone: - standalone: - cores_per_node: 20 - nodes: 1 -``` - -fate_on_eggroll：依据`EggRoll`集群实际部署情况进行配置，`nodes`表示`node manager`的机器数量，`cores_per_node`表示平均每台`node manager`机器CPU核数 - -```yaml -fate_on_eggroll: - clustermanager: - cores_per_node: 16 - nodes: 1 - rollsite: - host: 127.0.0.1 - port: 9370 -``` - -fate_on_spark：依据在`Spark`集群中配置给`FATE`集群的资源进行配置，`nodes`表示`Spark`节点数量，`cores_per_node`表示平均每个节点分配给`FATE`集群的CPU核数 - -```yaml -fate_on_spark: - spark: - # default use SPARK_HOME environment variable - home: - cores_per_node: 20 - nodes: 2 -``` - -注意：请务必确保在`Spark`集群分配了对应数量的资源于`FATE`集群，若`Spark`集群分配资源少于此处`FATE`所配置的资源，那么会出现可以提交`FATE`作业，但是`FATE Flow`将任务提交至`Spark`集群时，由于`Spark`集群资源不足，任务实际不执行 - -## 3. 作业申请资源配置 - -我们一般使用`task_cores`和`task_parallelism`进行配置作业申请资源，如： - -```json -{ -"job_parameters": { - "common": { - "job_type": "train", - "task_cores": 6, - "task_parallelism": 2, - "computing_partitions": 8, - "timeout": 36000 - } - } -} -``` - -作业申请的总资源为`task_cores` * `task_parallelism`，创建作业时，`FATE Flow`分发作业到各`party`时会依据上述配置、运行角色、本方使用引擎(通过`$FATE_PROJECT_BASE/conf/service_conf.yaml#default_engines`)，适配计算出实际参数，如下 - -## 4. 资源申请实际参数适配计算过程 - -- 计算`request_task_cores`: - - guest、host： - - `request_task_cores` = `task_cores` - - arbiter，考虑实际运行耗费极少资源： - - `request_task_cores` = 1 - -- 进一步计算`task_cores_per_node`： - - `task_cores_per_node"` = max(1, `request_task_cores` / `task_nodes`) - - - 若在上述`job_parameters`使用了`eggroll_run`或`spark_run`配置资源时，则`task_cores`配置无效；计算`task_cores_per_node`： - - `task_cores_per_node"` = eggroll_run[“eggroll.session.processors.per.node”] - - `task_cores_per_node"` = spark_run["executor-cores"] - -- 转换为适配引擎的参数(该参数会在运行任务时，提交到计算引擎识别)： - - fate_on_standalone/fate_on_eggroll: - - eggroll_run["eggroll.session.processors.per.node"] = `task_cores_per_node` - - fate_on_spark: - - spark_run["num-executors"] = `task_nodes` - - spark_run["executor-cores"] = `task_cores_per_node` - -- 最终计算结果可以查看job的`job_runtime_conf_on_party.json`，一般在`$FATE_PROJECT_BASE/jobs/$job_id/$role/$party_id/job_runtime_on_party_conf.json` - -## 5. 资源调度策略 - -- `total_cores`见上述[总资源配置](#2-总资源配置) -- `apply_cores`见上述[作业申请资源配置](#3-作业申请资源配置)，`apply_cores` = `task_nodes` * `task_cores_per_node` * `task_parallelism` -- 若所有参与方均申请资源成功(total_cores - apply_cores) > 0，则该作业申请资源成功 -- 若非所有参与方均申请资源成功，则发送资源回滚指令到已申请成功的参与方，该作业申请资源失败 - -## 6. 相关命令 - -{{snippet('cli/resource.zh.md', header=False)}} diff --git a/doc/fate_flow_server_operation.md b/doc/fate_flow_server_operation.md deleted file mode 100644 index 957575281..000000000 --- a/doc/fate_flow_server_operation.md +++ /dev/null @@ -1,13 +0,0 @@ -# Server Operation - -## 1. Description - -Starting from version `1.7.0`, we provide some maintenance functions for `FATE Flow Server`, which will be further enhanced in future versions. - -## 2. View version information - -{{snippet('cli/server.md', '### versions')}} - -## 3. Reload the configuration file - -{{snippet('cli/server.md', '### reload')}} \ No newline at end of file diff --git a/doc/fate_flow_server_operation.zh.md b/doc/fate_flow_server_operation.zh.md deleted file mode 100644 index d36ede18b..000000000 --- a/doc/fate_flow_server_operation.zh.md +++ /dev/null @@ -1,13 +0,0 @@ -# 服务端操作 - -## 1. 说明 - -从`1.7.0`版本开始, 提供`FATE Flow Server`的一些更新维护功能, 后续版本会进一步增强 - -## 2. 查看版本信息 - -{{snippet('cli/server.zh.md', '### versions')}} - -## 3. 重新加载配置文件 - -{{snippet('cli/server.zh.md', '### reload')}} \ No newline at end of file diff --git a/doc/fate_flow_service_registry.md b/doc/fate_flow_service_registry.md deleted file mode 100644 index 22583daa5..000000000 --- a/doc/fate_flow_service_registry.md +++ /dev/null @@ -1,32 +0,0 @@ -# Service Registry - -## 1. Description - -### 1.1 Model Registry - -FATE-Flow interacts with FATE-Serving through Apache ZooKeeper. If `use_registry` is enabled in the configuration, Flow registers model download URLs with ZooKeeper when it starts, and Serving can get the models through these URLs. - -Likewise, Serving registers its own address with ZooKeeper, which Flow will fetch to communicate with. If `use_registry` is not enabled, Flow will try to communicate with the set `servings` address in the configuration file. - -### 1.2 High Availability - -FATE-Flow implements automatic discovery of multiple nodes in the same party by registering its own IP and port with Apache ZooKeeper. - -## 2. Configuring the ZooKeeper service - -```yaml -zookeeper: - hosts: - - 127.0.0.1:2181 - use_acl: false - user: fate - password: fate -``` - -## 3. ZNode - -- FATE-Flow Model Registry: `/FATE-SERVICES/flow/online/transfer/providers` - -- FATE-Flow High Availability: `/FATE-COMPONENTS/fate-flow` - -- FATE-Serving: `/FATE-SERVICES/serving/online/publishLoad/providers` diff --git a/doc/fate_flow_service_registry.zh.md b/doc/fate_flow_service_registry.zh.md deleted file mode 100644 index 03ff9e04d..000000000 --- a/doc/fate_flow_service_registry.zh.md +++ /dev/null @@ -1,32 +0,0 @@ -# 服务注册中心 - -## 1. 说明 - -### 1.1 模型注册 - -FATE-Flow 通过 Apache ZooKeeper 与 FATE-Serving 交互，如果在配置中启用了 `use_registry`，则 Flow 在启动时会向 ZooKeeper 注册模型的下载 URL，Serving 可以通过这些 URL 获取模型。 - -同样，Serving 也会向 ZooKeeper 注册其自身的地址，Flow 会获取该地址以与之通信。 如果没有启用 `use_registry`，Flow 则会尝试与配置文件中的设置 `servings` 地址通信。 - -### 1.2 高可用 - -FATE-Flow 通过向 Apache ZooKeeper 注册自身的 IP 和端口实现同一 party 内多节点的自动发现。 - -## 2. 配置 ZooKeeper 服务 - -```yaml -zookeeper: - hosts: - - 127.0.0.1:2181 - use_acl: false - user: fate - password: fate -``` - -## 3. ZNode - -- FATE-Flow 模型注册: `/FATE-SERVICES/flow/online/transfer/providers` - -- FATE-Flow 高可用: `/FATE-COMPONENTS/fate-flow` - -- FATE-Serving: `/FATE-SERVICES/serving/online/publishLoad/providers` diff --git a/doc/fate_flow_tracking.md b/doc/fate_flow_tracking.md deleted file mode 100644 index 0190872db..000000000 --- a/doc/fate_flow_tracking.md +++ /dev/null @@ -1,49 +0,0 @@ -# Data Flow Tracking - -## 1. Description - -## 2. Task output indicators - -## 2.1 List of metrics - -{{snippet('cli/tracking.md', '### metrics')}} - -### 2.2 All metrics - -{{snippet('cli/tracking.md', '### metric-all')}} - -## 3. Task run parameters - -{{snippet('cli/tracking.md', '### parameters')}} - -## 4. Task output data - -### 4.1 Download output data - -{{snippet('cli/tracking.md', '### output-data')}} - -### 4.2 Get the name of the data table where the output data is stored - -{{snippet('cli/tracking.md', '### output-data-table')}} - -## 5. Task output model - -{{snippet('cli/tracking.md', '### output-model')}} - -## 6. Task output summary - -{{snippet('cli/tracking.md', '### get-summary')}} - -## 7. Dataset usage tracking - -Tracing source datasets and their derived datasets, such as component task output datasets - -### 7.1 Source table query - -{{snippet('cli/tracking.md', '### tracking-source')}} - -### 7.2 Querying with table tasks - -{{snippet('cli/tracking.md', '### tracking-job')}} - -## 8. Developing the API \ No newline at end of file diff --git a/doc/fate_flow_tracking.zh.md b/doc/fate_flow_tracking.zh.md deleted file mode 100644 index 358e9dd2e..000000000 --- a/doc/fate_flow_tracking.zh.md +++ /dev/null @@ -1,49 +0,0 @@ -# 数据流动追踪 - -## 1. 说明 - -## 2. 任务输出指标 - -### 2.1 指标列表 - -{{snippet('cli/tracking.zh.md', '### metrics')}} - -### 2.2 所有指标 - -{{snippet('cli/tracking.zh.md', '### metric-all')}} - -## 3. 任务运行参数 - -{{snippet('cli/tracking.zh.md', '### parameters')}} - -## 4. 任务输出数据 - -### 4.1 下载输出数据 - -{{snippet('cli/tracking.zh.md', '### output-data')}} - -### 4.2 获取输出数据存放数据表名称 - -{{snippet('cli/tracking.zh.md', '### output-data-table')}} - -## 5. 任务输出模型 - -{{snippet('cli/tracking.zh.md', '### output-model')}} - -## 6. 任务输出摘要 - -{{snippet('cli/tracking.zh.md', '### get-summary')}} - -## 7. 数据集使用追踪 - -追踪源数据集及其衍生数据集，如组件任务输出数据集 - -### 7.1 源表查询 - -{{snippet('cli/tracking.zh.md', '### tracking-source')}} - -### 7.2 用表任务查询 - -{{snippet('cli/tracking.zh.md', '### tracking-job')}} - -## 8. 开发API diff --git a/doc/images/container_load.png b/doc/images/container_load.png new file mode 100644 index 000000000..7146b605a Binary files /dev/null and b/doc/images/container_load.png differ diff --git a/doc/images/federationml_schedule.png b/doc/images/federationml_schedule.png new file mode 100644 index 000000000..aa070a488 Binary files /dev/null and b/doc/images/federationml_schedule.png differ diff --git a/doc/images/log.png b/doc/images/log.png new file mode 100644 index 000000000..bf2e0c737 Binary files /dev/null and b/doc/images/log.png differ diff --git a/doc/images/open_flow.png b/doc/images/open_flow.png new file mode 100644 index 000000000..538f49546 Binary files /dev/null and b/doc/images/open_flow.png differ diff --git a/doc/images/pull.png b/doc/images/pull.png new file mode 100644 index 000000000..78f0c2a3a Binary files /dev/null and b/doc/images/pull.png differ diff --git a/doc/images/push.png b/doc/images/push.png new file mode 100644 index 000000000..e7cbf5ae4 Binary files /dev/null and b/doc/images/push.png differ diff --git a/doc/images/schedule_for_callback.png b/doc/images/schedule_for_callback.png new file mode 100644 index 000000000..1c9e77b34 Binary files /dev/null and b/doc/images/schedule_for_callback.png differ diff --git a/doc/images/schedule_for_component.png b/doc/images/schedule_for_component.png new file mode 100644 index 000000000..8f35e3ad4 Binary files /dev/null and b/doc/images/schedule_for_component.png differ diff --git a/doc/images/schedule_for_poll.png b/doc/images/schedule_for_poll.png new file mode 100644 index 000000000..cb5aba946 Binary files /dev/null and b/doc/images/schedule_for_poll.png differ diff --git a/doc/quick_start.md b/doc/quick_start.md new file mode 100644 index 000000000..bfd79833c --- /dev/null +++ b/doc/quick_start.md @@ -0,0 +1,332 @@ +# Quick Start + +## 1. Environment Setup +You can choose one of the following three deployment modes based on your requirements: + +### 1.1 Pypi Package Installation +Note: This mode operates in a single-machine mode. + +#### 1.1.1 Installation +- Prepare and install [conda](https://docs.conda.io/projects/miniconda/en/latest/) environment. +- Create a virtual environment: +```shell +# FATE requires Python >= 3.8 +conda create -n fate_env python=3.8 +conda activate fate_env +``` +- Install FATE Flow and related dependencies: +```shell +pip install fate_client[fate,fate_flow]==2.0.0.b0 +``` + +#### 1.1.2 Service Initialization +```shell +fate_flow init --ip 127.0.0.1 --port 9380 --home $HOME_DIR +``` +- `ip`: The IP address where the service runs. +- `port`: The HTTP port the service runs on. +- `home`: The data storage directory, including data, models, logs, job configurations, and SQLite databases. + +#### 1.1.3 Service Start/Stop +```shell +fate_flow status/start/stop/restart +``` + +### 1.2 Standalone Deployment +Refer to [Standalone Deployment](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/deploy/standalone-deploy/README.zh.md). + +### 1.3 Cluster Deployment +Refer to [Allinone Deployment](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/deploy/cluster-deploy/allinone/fate-allinone_deployment_guide.zh.md). + +## 2. User Guide +FATE provides client tools including SDK, CLI, and Pipeline. If you don't have FATE Client deployed in your environment, you can download it using `pip install fate_client`. The following operations are based on CLI. + +### 2.1 Data Upload +In version 2.0-beta, data uploading is a two-step process: + +- **upload**: Uploads data to FATE-supported storage services. +- **transformer**: Transforms data into a DataFrame. + +#### 2.1.1 upload +##### 2.1.1.1 Configuration and Data +- Upload configuration can be found at [examples-upload](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/upload), and the data is located at [upload-data](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/data). +- You can also use your own data and modify the "meta" information in the upload configuration. + +##### 2.1.1.2 Upload Guest Data +```shell +flow data upload -c examples/upload/upload_guest.json +``` +- Record the returned "name" and "namespace" for use in the transformer phase. + +##### 2.1.1.3 Upload Host Data +```shell +flow data upload -c examples/upload/upload_host.json +``` +- Record the returned "name" and "namespace" for use in the transformer phase. + +##### 2.1.1.4 Upload Result +```json +{ + "code": 0, + "data": { + "name": "36491bc8-3fef-11ee-be05-16b977118319", + "namespace": "upload" + }, + "job_id": "202308211451535620150", + "message": "success" +} +``` +Where "namespace" and "name" identify the data in FATE for future reference in the transformer phase. + +##### 2.1.1.5 Data Query +Since upload is an asynchronous operation, you need to confirm if it was successful before proceeding to the next step. +```shell +flow table query --namespace upload --name 36491bc8-3fef-11ee-be05-16b977118319 +``` +If the returned code is 0, the upload was successful. + +#### 2.1.2 Transformer +##### 2.1.2.1 Configuration +- Transformer configuration can be found at [examples-transformer](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/transformer). + +##### 2.1.2.2 Transform Guest Data +- Configuration path: examples/transformer/transformer_guest.json +- Modify the "namespace" and "name" in the "data_warehouse" section to match the output from the guest data upload. +```shell +flow data transformer -c examples/transformer/transformer_guest.json +``` + +##### 2.1.2.3 Transform Host Data +- Configuration path: examples/transformer/transformer_host.json +- Modify the "namespace" and "name" in the "data_warehouse" section to match the output from the host data upload. +```shell +flow data transformer -c examples/transformer/transformer_host.json +``` + +##### 2.1.2.4 Transformer Result +```json +{ + "code": 0, + "data": { + "name": "breast_hetero_guest", + "namespace": "experiment" + }, + "job_id": "202308211557455662860", + "message": "success" +} +``` +Where "namespace" and "name" identify the data in FATE for future modeling jobs. + +##### 2.1.2.5 Check if Data Upload Was Successful +Since the transformer is also an asynchronous operation, you need to confirm if it was successful before proceeding. +```shell +flow table query --namespace experiment --name breast_hetero_guest +``` +```shell +flow table query --namespace experiment --name breast_hetero_host +``` +If the returned code is 0, the upload was successful. + +### 2.2 Starting FATE Jobs +#### 2.2.1 Submitting a Job +Once your data is prepared, you can start submitting jobs to FATE Flow: + +- The configuration for training jobs can be found in [lr-train](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/lr/train_lr.yaml). +- The configuration for prediction jobs can be found in [lr-predict](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/lr/predict_lr.yaml). To use it, modify the "dag.conf.model_warehouse" to point to the output model of your training job. +- In the training and prediction job configurations, the site IDs are set to "9998" and "9999." If your deployment environment is the cluster version, you need to replace them with the actual site IDs. For the standalone version, you can use the default configuration. +- If you want to use your own data, you can change the "namespace" and "name" of "data_warehouse" for both the guest and host in the configuration. +- To submit a job, use the following command: +```shell +flow job submit -c examples/lr/train_lr.yaml +``` +- A successful submission will return the following result: +```json +{ + "code": 0, + "data": { + "model_id": "202308211911505128750", + "model_version": "0" + }, + "job_id": "202308211911505128750", + "message": "success" +} +``` +The "data" section here contains the output model of the job. + +#### 2.2.2 Querying a Job +While a job is running, you can check its status using the query command: +```shell +flow job query -j $job_id +``` + +#### 2.2.3 Stopping a Job +During job execution, you can stop the current job using the stop command: +```shell +flow job stop -j $job_id +``` + +#### 2.2.4 Rerunning a Job +If a job fails during execution, you can rerun it using the rerun command: +```shell +flow job rerun -j $job_id +``` + +### 2.3 Obtaining Job Outputs +Job outputs include data, models, and metrics. + +#### 2.3.1 Output Metrics +To query output metrics, use the following command: +```shell +flow output query-metric -j $job_id -r $role -p $party_id -tn $task_name +``` +For example, if you used the training DAG from above, you can use `flow output query-metric -j 202308211911505128750 -r arbiter -p 9998 -tn lr_0` to query metrics. +The query result will look like this: +```json +{ + "code": 0, + "data": [ + { + "data": [ + { + "metric": [ + 0.0 + ], + "step": 0, + "timestamp": 1692616428.253495 + } + ], + "groups": [ + { + "index": null, + "name": "default" + }, + { + "index": null, + "name": "train" + } + ], + "name": "lr_loss", + "step_axis": "iterations", + "type": "loss" + }, + { + "data": [ + { + "metric": [ + -0.07785049080848694 + ], + "step": 1, + "timestamp": 1692616432.9727712 + } + ], + "groups": [ + { + "index": null, + "name": "default" + }, + { + "index": null, + "name": "train" + } + ], + "name": "lr_loss", + "step_axis": "iterations", + "type": "loss" + } + ], + "message": "success" +} +``` + +#### 2.3.2 Output Models +##### 2.3.2.1 Querying Models +To query output models, use the following command: +```shell +flow output query-model -j $job_id -r $role -p $party_id -tn $task_name +``` +For example, if you used the training DAG from above, you can use `flow output query-model -j 202308211911505128750 -r host -p 9998 -tn lr_0` to query models. +The query result will be similar to this: + +```json +{ + "code": 0, + "data": [ + { + "model": { + "file": "202308211911505128750_host_9998_lr_0", + "namespace": "202308211911505128750_host_9998_lr_0" + }, + "name": "HeteroLRHost_9998_0", + "namespace": "202308211911505128750_host_9998_lr_0", + "role": "host", + "party_id": "9998", + "work_mode": 1 + } + ], + "message": "success" +} +``` + +##### 2.3.2.2 Downloading Models +To download models, use the following command: +```shell +flow output download-model -j $job_id -r $role -p $party_id -tn $task_name -o $download_dir +``` +For example, if you used the training DAG from above, you can use `flow output download-model -j 202308211911505128750 -r host -p 9998 -tn lr_0 -o ./` to download the model. +The download result will be similar to this: + +```json +{ + "code": 0, + "directory": "./output_model_202308211911505128750_host_9998_lr_0", + "message": "download success, please check the path: ./output_model_202308211911505128750_host_9998_lr_0" +} +``` + +#### 2.3.3 Output Data +##### 2.3.3.1 Querying Data Tables +To query output data tables, use the following command: +```shell +flow output query-data-table -j $job_id -r $role -p $party_id -tn $task_name +``` +For example, if you used the training DAG from above, you can use `flow output query-data-table -j 202308211911505128750 -r host -p 9998 -tn binning_0` to query data tables. +The query result will be similar to this: + +```json +{ + "train_output_data": [ + { + "name": "9e28049c401311ee85c716b977118319", + "namespace": "202308211911505128750_binning_0" + } + ] +} +``` + +##### 2.3.3.2 Preview Data +```shell +flow output display-data -j $job_id -r $role -p $party_id -tn $task_name +``` +To preview output data using the above training DAG submission, you can use the following command: `flow output display-data -j 202308211911505128750 -r host -p 9998 -tn binning_0`. + +##### 2.3.3.3 Download Data +```shell +flow output download-data -j $job_id -r $role -p $party_id -tn $task_name -o $download_dir +``` +To download output data using the above training DAG submission, you can use the following command: `flow output download-data -j 202308211911505128750 -r guest -p 9999 -tn lr_0 -o ./`. + +The download result will be as follows: +```json +{ + "code": 0, + "directory": "./output_data_202308211911505128750_guest_9999_lr_0", + "message": "download success, please check the path: ./output_data_202308211911505128750_guest_9999_lr_0" +} +``` + +## 3. More Documentation +- [Restful-api](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/doc/swagger/swagger.yaml) +- [CLI](https://github.com/FederatedAI/FATE-Client/tree/v2.0.0-beta/python/fate_client/flow_cli/build/doc) +- [Pipeline](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/doc/tutorial) +- [FATE Quick Start](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/doc/2.0/quick_start.md) +- [FATE Algorithms](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/doc/2.0/fate) \ No newline at end of file diff --git a/doc/quick_start.zh.md b/doc/quick_start.zh.md new file mode 100644 index 000000000..90176526b --- /dev/null +++ b/doc/quick_start.zh.md @@ -0,0 +1,562 @@ +# 快速入门 + +## 1. 环境部署 +以下三种模式可根据需求自行选择一种 +### 1.1 Pypi包安装 +说明：此方式的运行模式为单机模式 +#### 1.1.1 安装 +- [conda](https://docs.conda.io/projects/miniconda/en/latest/)环境准备及安装 +- 创建虚拟环境 +```shell +# fate的运行环境为python>=3.8 +conda create -n fate_env python=3.8 +conda activate fate_env +``` +- 安装fate flow及相关依赖 +```shell +pip install fate_client[fate,fate_flow]==2.0.0.b0 +``` + +#### 1.1.2 服务初始化 +```shell +fate_flow init --ip 127.0.0.1 --port 9380 --home $HOME_DIR +``` +- ip: 服务运行ip +- port：服务运行时的http端口 +- home: 数据存储目录。主要包括：数据/模型/日志/作业配置/sqlite.db等内容 + +#### 1.1.3 服务启停 +```shell +fate_flow status/start/stop/restart +``` + +### 1.2 单机版部署 +参考[单机版部署](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/deploy/standalone-deploy/README.zh.md) + +### 1.3 集群部署 +参考[allinone部署](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/deploy/cluster-deploy/allinone/fate-allinone_deployment_guide.zh.md) + +## 2. 使用指南 +fate提供的客户端包括SDK、CLI和Pipeline，若你的环境中没有部署FATE Client,可以使用`pip install fate_client`下载，以下的使用操作均基于cli编写。 + +### 2.1 数据上传 +在2.0-beta版本中，数据上传分为两步： +- upload: 将数据上传到FATE支持存储服务中 +- transformer: 将数据转化成dataframe +#### 2.1.1 upload +#### 2.1.1.1 配置及数据 + - 上传配置位于[examples-upload](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/upload)，上传数据位于[upload-data](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/data) + - 你也可以使用自己的数据，并修改upload配置中的"meta"信息。 +#### 2.1.1.2 上传guest方数据 +```shell +flow data upload -c examples/upload/upload_guest.json +``` +- 需要记录返回的name和namespace，作为transformer的参数。 +#### 2.1.1.3 上传host方数据 +```shell +flow data upload -c examples/upload/upload_host.json +``` +- 需要记录返回的name和namespace，作为transformer的参数。 +#### 2.1.1.4 上传结果 +```json +{ + "code": 0, + "data": { + "name": "36491bc8-3fef-11ee-be05-16b977118319", + "namespace": "upload" + }, + "job_id": "202308211451535620150", + "message": "success" +} +``` +其中"namespace"和"name"是这份数据在fate中的标识，以便下面后续transformer阶段使用时可直接引用。 + +#### 2.1.1.5 数据查询 +因为upload为异步操作，需要确认是否上传成功才可进行后续操作。 +```shell +flow table query --namespace upload --name 36491bc8-3fef-11ee-be05-16b977118319 +``` +上传成功信息如下： +```json +{ + "code": 0, + "data": { + "count": 569, + "data_type": "table", + "engine": "standalone", + "meta": { + "delimiter": ",", + "dtype": "'float32", + "header": "extend_sid,id,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15,x16,x17,x18,x19", + "input_format": "dense", + "label_type": "int", + "match_id_name": "id", + "match_id_range": 0, + "sample_id_name": "extend_sid", + "tag_value_delimiter": ":", + "tag_with_value": false, + "weight_type": "float32" + }, + "name": "36491bc8-3fef-11ee-be05-16b977118319", + "namespace": "upload", + "path": "xxx", + "source": { + "component": "upload", + "output_artifact_key": "data", + "output_index": null, + "party_task_id": "", + "task_id": "", + "task_name": "upload" + } + }, + "message": "success" +} + +``` +若返回的code为0即为上传成功。 + +#### 2.1.2 transformer +#### 2.1.2.1 配置 + - transformer配置位于[examples-transformer](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/transformer) +#### 2.1.2.2 transformer guest +- 配置路径位于： examples/transformer/transformer_guest.json +- 修改配置中"data_warehouse"的"namespace"和"name"：上面upload guest阶段的输出 +```shell +flow data transformer -c examples/transformer/transformer_guest.json +``` +#### 2.1.2.3 transformer host +- 配置路径位于： examples/transformer/transformer_host.json +- 修改配置中"data_warehouse"的"namespace"和"name"：上面upload host阶段的输出 +```shell +flow data transformer -c examples/transformer/transformer_host.json +``` +#### 2.1.2.4 transformer结果 +```json +{ + "code": 0, + "data": { + "name": "breast_hetero_guest", + "namespace": "experiment" + }, + "job_id": "202308211557455662860", + "message": "success" +} +``` +其中"namespace"和"name"是这份数据在fate中的标识，后续建模作业中使用。 + +#### 2.1.2.5 查看数据是否上传成功 + +因为transformer也是异步操作，需要确认是否上传成功才可进行后续操作。 +```shell +flow table query --namespace experiment --name breast_hetero_guest +``` +```shell +flow table query --namespace experiment --name breast_hetero_host +``` +若返回的code为0即为上传成功。 + +### 2.2 开始FATE作业 +#### 2.2.1 提交作业 +当你的数据准备好后，可以开始提交作业给FATE Flow： +- 训练job配置example位于[lr-train](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/lr/train_lr.yaml); +- 预测job配置example位于[lr-predict](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/examples/lr/predict_lr.yaml);预测任务需要修改"dag.conf.model_warehouse"成训练作业的输出模型。 +- 训练和预测job配置中站点id为"9998"和"9999"。如果你的部署环境为集群版，需要替换成真实的站点id；单机版可使用默认配置。 +- 如果想要使用自己的数据，可以更改配置中guest和host的data_warehouse的namespace和name +- 提交作业的命令为: +```shell +flow job submit -c examples/lr/train_lr.yaml +``` +- 提交成功返回结果: +```json +{ + "code": 0, + "data": { + "model_id": "202308211911505128750", + "model_version": "0" + }, + "job_id": "202308211911505128750", + "message": "success" +} + +``` +这里的"data"内容即为该作业的输出模型。 + +#### 2.2.2 查询作业 +在作业的运行过程时，你可以通过查询命令获取作业的运行状态 +```shell +flow job query -j $job_id +``` + +#### 2.2.3 停止作业 +在作业的运行过程时，你可以通过停止作业命令来终止当前作业 +```shell +flow job stop -j $job_id +``` + +#### 2.2.4 重跑作业 +在作业的运行过程时，如果运行失败，你可以通过重跑命令来重跑当前作业 +```shell +flow job rerun -j $job_id +``` + +### 2.3 获取作业输出结果 +作业的输出包括数据、模型和指标 +#### 2.3.1 输出指标 +查询输出指标命令： +```shell +flow output query-metric -j $job_id -r $role -p $party_id -tn $task_name +``` +如使用上面的训练dag提交任务，可以使用`flow output query-metric -j 202308211911505128750 -r arbiter -p 9998 -tn lr_0`查询。 +查询结果如下: +```json +{ + "code": 0, + "data": [ + { + "data": [ + { + "metric": [ + 0.0 + ], + "step": 0, + "timestamp": 1692616428.253495 + } + ], + "groups": [ + { + "index": null, + "name": "default" + }, + { + "index": null, + "name": "train" + } + ], + "name": "lr_loss", + "step_axis": "iterations", + "type": "loss" + }, + { + "data": [ + { + "metric": [ + -0.07785049080848694 + ], + "step": 1, + "timestamp": 1692616432.9727712 + } + ], + "groups": [ + { + "index": null, + "name": "default" + }, + { + "index": null, + "name": "train" + } + ], + "name": "lr_loss", + "step_axis": "iterations", + "type": "loss" + } + ], + "message": "success" +} + +``` + + +#### 2.3.2 输出模型 +##### 2.3.2.1 查询模型 +```shell +flow output query-model -j $job_id -r $role -p $party_id -tn $task_name +``` +如使用上面的训练dag提交任务，可以使用`flow output query-model -j 202308211911505128750 -r host -p 9998 -tn lr_0`查询。 +查询结果如下： +```json +{ + "code": 0, + "data": { + "output_model": { + "data": { + "estimator": { + "end_epoch": 10, + "is_converged": false, + "lr_scheduler": { + "lr_params": { + "start_factor": 0.7, + "total_iters": 100 + }, + "lr_scheduler": { + "_get_lr_called_within_step": false, + "_last_lr": [ + 0.07269999999999996 + ], + "_step_count": 10, + "base_lrs": [ + 0.1 + ], + "end_factor": 1.0, + "last_epoch": 9, + "start_factor": 0.7, + "total_iters": 100, + "verbose": false + }, + "method": "linear" + }, + "optimizer": { + "alpha": 0.001, + "l1_penalty": false, + "l2_penalty": true, + "method": "sgd", + "model_parameter": [ + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ] + ], + "model_parameter_dtype": "float32", + "optim_param": { + "lr": 0.1 + }, + "optimizer": { + "param_groups": [ + { + "dampening": 0, + "differentiable": false, + "foreach": null, + "initial_lr": 0.1, + "lr": 0.07269999999999996, + "maximize": false, + "momentum": 0, + "nesterov": false, + "params": [ + 0 + ], + "weight_decay": 0 + } + ], + "state": {} + } + }, + "param": { + "coef_": [ + [ + -0.10828543454408646 + ], + [ + -0.07341302931308746 + ], + [ + -0.10850320011377335 + ], + [ + -0.10066638141870499 + ], + [ + -0.04595951363444328 + ], + [ + -0.07001449167728424 + ], + [ + -0.08949052542448044 + ], + [ + -0.10958756506443024 + ], + [ + -0.04012322425842285 + ], + [ + 0.02270071767270565 + ], + [ + -0.07198350876569748 + ], + [ + 0.00548586156219244 + ], + [ + -0.06599288433790207 + ], + [ + -0.06410090625286102 + ], + [ + 0.016374297440052032 + ], + [ + -0.01607361063361168 + ], + [ + -0.011447405442595482 + ], + [ + -0.04352564364671707 + ], + [ + 0.013161249458789825 + ], + [ + 0.013506329618394375 + ] + ], + "dtype": "float32", + "intercept_": null + } + } + }, + "meta": { + "batch_size": null, + "epochs": 10, + "init_param": { + "fill_val": 0.0, + "fit_intercept": false, + "method": "zeros", + "random_state": null + }, + "label_count": false, + "learning_rate_param": { + "method": "linear", + "scheduler_params": { + "start_factor": 0.7, + "total_iters": 100 + } + }, + "optimizer_param": { + "alpha": 0.001, + "method": "sgd", + "optimizer_params": { + "lr": 0.1 + }, + "penalty": "l2" + }, + "ovr": false + } + } + }, + "message": "success" +} + +``` + +##### 2.3.2.2 下载模型 +```shell +flow output download-model -j $job_id -r $role -p $party_id -tn $task_name -o $download_dir +``` +如使用上面的训练dag提交任务，可以使用`flow output download-model -j 202308211911505128750 -r host -p 9998 -tn lr_0 -o ./`下载。 +下载结果如下： +```json +{ + "code": 0, + "directory": "./output_model_202308211911505128750_host_9998_lr_0", + "message": "download success, please check the path: ./output_model_202308211911505128750_host_9998_lr_0" +} + + +``` + + +#### 2.3.3 输出数据 +##### 2.3.3.1 查询数据表 +```shell +flow output query-data-table -j $job_id -r $role -p $party_id -tn $task_name +``` +如使用上面的训练dag提交任务，可以使用`flow output query-data-table -j 202308211911505128750 -r host -p 9998 -tn binning_0`查询。 +查询结果如下： +```json +{ + "train_output_data": [ + { + "name": "9e28049c401311ee85c716b977118319", + "namespace": "202308211911505128750_binning_0" + } + ] +} +``` + +##### 2.3.3.2 预览数据 +```shell +flow output display-data -j $job_id -r $role -p $party_id -tn $task_name +``` +如使用上面的训练dag提交任务，可以使用`flow output display-data -j 202308211911505128750 -r host -p 9998 -tn binning_0`预览输出数据。 + +##### 2.3.3.3 下载数据 +```shell +flow output download-data -j $job_id -r $role -p $party_id -tn $task_name -o $download_dir +``` +如使用上面的训练dag提交任务，可以使用`flow output download-data -j 202308211911505128750 -r guest -p 9999 -tn lr_0 -o ./`下载输出数据。 +下载结果如下： +```json +{ + "code": 0, + "directory": "./output_data_202308211911505128750_guest_9999_lr_0", + "message": "download success, please check the path: ./output_data_202308211911505128750_guest_9999_lr_0" +} + +``` + +## 3.更多文档 +- [Restful-api](https://github.com/FederatedAI/FATE-Flow/tree/v2.0.0-beta/doc/swagger/swagger.yaml) +- [CLI](https://github.com/FederatedAI/FATE-Client/tree/v2.0.0-beta/python/fate_client/flow_cli/build/doc) +- [Pipeline](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/doc/tutorial) +- [FATE快速开始](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/doc/2.0/quick_start.md) +- [FATE算法](https://github.com/FederatedAI/FATE/tree/v2.0.0-beta/doc/2.0/fate) diff --git a/doc/swagger/swagger.yaml b/doc/swagger/swagger.yaml index 0855feb68..a20e12ed7 100644 --- a/doc/swagger/swagger.yaml +++ b/doc/swagger/swagger.yaml @@ -1,4382 +1,2074 @@ -openapi: 3.0.3 -info: - version: '1.8.0' - title: Fate Flow - +swagger: '2.0' +basePath: /v2 paths: - '/data/upload': + /client/client/create: post: - summary: upload - tags: - - data-access - parameters: - - in: query - name: id_delimiter - description: data delimiter - required: false - schema: - type: string - example: "," - - in: query - name: head - description: data head - required: true + responses: + '200': + description: Success schema: - type: integer - example: 0, 1 - - in: query - name: partition - description: compoting table partitions + $ref: '#/definitions/ResponseClientInfo' + operationId: post_create_client_app + parameters: + - name: payload required: true + in: body schema: - type: integer - example: 16, ... - - in: query - name: table_name - description: fate table name - required: true + $ref: '#/definitions/ReqAppName' + tags: + - client + /client/client/delete: + post: + responses: + '200': + description: Success schema: - type: string - example: breast_hetero_guest - - in: query - name: namespace - description: fate table namespace + $ref: '#/definitions/ResponseStatus' + operationId: post_delete_client_app + parameters: + - name: payload required: true + in: body schema: - type: string - example: experiment - - in: query - name: storage_engine - description: data storage engin - required: false + $ref: '#/definitions/ReqAppId' + tags: + - client + /client/client/query: + get: + responses: + '200': + description: Success schema: - type: string - example: eggroll, localfs, hdfs, ... - - in: query - name: destory - description: destory old table and upload new table - required: false + $ref: '#/definitions/ResponseClientInfo' + operationId: get_client_QueryClientApp + parameters: + - required: false + description: App ID for the client + in: query + name: app_id + type: string + - required: false + description: App name for the client + in: query + name: app_name + type: string + tags: + - client + /client/partner/create: + post: + responses: + '200': + description: Success schema: - type: integer - example: 0, 1 - - in: query - name: extend_sid - description: extend sid to first column - required: false + $ref: '#/definitions/ResponsePartnerInfo' + operationId: post_create_partner_app + parameters: + - name: payload + required: true + in: body schema: - type: integer - example: 0, 1 - requestBody: - required: true - content: - application/octet-stream: - schema: - type: string + $ref: '#/definitions/ReqPartnerCreate' + tags: + - client + /client/partner/delete: + post: responses: '200': - description: upload success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "board_url": "http://xxx:8080/index.html#/dashboard?job_id=xxx&role=local&party_id=0", - "code": 0, - "dsl_path": "/data/projects/fate/fateflow/jobs/xxx/job_dsl.json", - "job_id": xxx, - "logs_directory": "/data/projects/fate/fateflow/logs/xxx", - "message": "success", - "model_info": { - "model_id": "local-0#model", - "model_version": xxx - }, - "namespace": "experiment", - "pipeline_dsl_path": "/data/projects/fate/fateflow/jobs/xxx/pipeline_dsl.json", - "runtime_conf_on_party_path": "/data/projects/fate/fateflow/jobs/xxx/local/0/job_runtime_on_party_conf.json", - "runtime_conf_path": "/data/projects/fate/fateflow/jobs/xxx/job_runtime_conf.json", - "table_name": "breast_hetero_guest", - "train_runtime_conf_path": "/data/projects/fate/fateflow/jobs/xxx/train_runtime_conf.json" - } - '404': - description: upload failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: required parameters are missing - - '/data/download': - post: - summary: download data + description: Success + schema: + $ref: '#/definitions/ResponseStatus' + operationId: post_delete_partner_app + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqPartyId' tags: - - data-access - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - table_name - - namespace - - output_path - properties: - table_name: - type: string - example: breast_hetero_guest - namespace: - type: string - example: experiment - output_path: - type: string - example: /data/projects/fate/fateflow/experiment_download_breast_guest.csv - delimiter: - type: string - example: "," + - client + /client/partner/query: + get: responses: '200': - description: download success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "board_url": "http://xxx:8080/index.html#/dashboard?job_id=xxx&role=local&party_id=0", - "code": 0, - "dsl_path": "/data/projects/fate/fateflow/jobs/xxx/job_dsl.json", - "job_id": xxx, - "logs_directory": "/data/projects/fate/fateflow/logs/xxx", - "message": "success", - "model_info": { - "model_id": "local-0#model", - "model_version": xxx - }, - "namespace": "experiment", - "pipeline_dsl_path": "/data/projects/fate/fateflow/jobs/xxx/pipeline_dsl.json", - "runtime_conf_on_party_path": "/data/projects/fate/fateflow/jobs/xxx/local/0/job_runtime_on_party_conf.json", - "runtime_conf_path": "/data/projects/fate/fateflow/jobs/xxx/job_runtime_conf.json", - "table_name": "breast_hetero_guest", - "train_runtime_conf_path": "/data/projects/fate/fateflow/jobs/xxx/train_runtime_conf.json" - } - '404': - description: download failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: required parameters are missing - - '/data/upload/history': - post: - summary: history of upload job info + description: Success + schema: + $ref: '#/definitions/ResponseClientInfo' + operationId: get_client_QueryPartnerApp + parameters: + - required: false + description: Site ID + in: query + name: party_id + type: string tags: - - data-access - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - job_id: - type: string - example: 202103241706521313480 - limit: - type: integer - description: limit output - example: 1 + - client + /client/site/create: + post: responses: '200': - description: get success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: array - items: - type: object - example: - { - "202103241706521313480": { - "notes": "", - "schema": { - "header": "y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9", - "sid": "id" - }, - "upload_info": { - "namespace": "experiment", - "partition": 4, - "table_name": "breast_hetero_guest", - "upload_count": 569 - } - } - } - '404': - description: get failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 100 - retmsg: - type: string - example: server error - - '/table/bind': - post: - summary: bind a storage address to fate table + description: Success + schema: + $ref: '#/definitions/ResponseClientInfo' + operationId: post_create_site_app + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqPartyId' tags: - - table - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - engine - - address - - namespace - - name - - head - - id_delimiter - - partitions - properties: - engine: - type: string - example: mysql - name: - type: string - example: breast_hetero_guest - namespace: - type: string - example: experiment - address: - type: object - description: storage address - example: - user: fate - passwd: fate - host: 127.0.0.1 - port: 3306 - db: xxx - name: xxx - partitions: - type: integer - description: fate computing table partitions - example: 16 - head: - type: integer - description: 1 means data have head - example: 0,1 - id_delimiter: - type: string - description: data table or intermediate storage table delimiter - example: "," - in_serialized: - type: integer - description: data serialized, standlone/eggroll/mysql/path storage default 1, others default 0 - example: 0, 1 - drop: - type: integer - description: if table is exist, will delete it - example: 0,1 - id_column: - type: string - example: "id" - feature_column: - type: string - description: delimited by "," - example: x1,x2,x3 + - client + /client/site/delete: + post: responses: '200': - description: bind table success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - properties: - table_name: - type: string - example: breast_hetero_guest - namespace: - type: string - example: experiment - '404': - description: bind table failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 100 - retmsg: - type: string - example: engine xxx address xxx check failed - - '/table/delete': - post: - summary: delete fate table + description: Success + schema: + $ref: '#/definitions/ResponseStatus' + operationId: post_delete_site_app + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqPartyId' tags: - - table - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - table_name - - namespace - properties: - table_name: - type: string - example: breast_hetero_guest - namespace: - type: string - example: experiment + - client + /client/site/query: + get: responses: '200': - description: delete table success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - properties: - table_name: - type: string - example: breast_hetero_guest - namespace: - type: string - example: experiment - '404': - description: delete table failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no find table - - '/table/list': - post: - summary: get job all tables + description: Success + schema: + $ref: '#/definitions/ResponseClientInfo' + operationId: get_client_QuerySiteApp + parameters: + - required: true + description: Site ID + in: query + name: party_id + type: string tags: - - table - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - job_id - - role - - party_id - properties: - job_id: - type: string - example: 202101251515021092240 - role: - type: string - example: guest - party_id: - type: string - example: 10000 + - client + /data/component/dataframe/transformer: + post: responses: '200': - description: get tables success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "DataIO_0": - { - "input": - { - "Reader_0.data_0": - { - "name": xxx, - "namespace": xxx - } - }, - "output": - { - "data_0": - { - "name": xxx, - "namespace": xxx - } - } - }, - "Intersection_0": - { - "input": - { - "DataIO_0.data_0": - { - "name": xxx, - "namespace": xxx - } - }, - "output": - { - "data_0": - { - "name": xxx, - "namespace": xxx - } - } - }, - "Reader_0": - { - "input": - { - "table": - { - "name": xxx, - "namespace": "xxxx" - } - }, - "output": - { - "data_0": - { - "name": xxx, - "namespace": xxx - } - } - } - } - '404': - description: delete table failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no find table - - '/table/table_info': - post: - summary: query table info + description: Success + schema: + $ref: '#/definitions/ResponseUploadData' + operationId: post_transformer_data + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqTransformerData' tags: - - table - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - table_name - - namespace - properties: - table_name: - type: string - example: breast_hetero_guest - namespace: - type: string - example: experiment + - data + /data/component/download: + post: responses: '200': - description: get tables success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "address": - {}, - "count": 569, - "exist": 1, - "namespace": "experiment", - "partition": 16, - "schema": - { - "header": "id,y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9", - "sid": "id" - }, - "table_name": "breast_hetero_guest" - } - '404': - description: query table failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no find table - - '/table/tracking/source': - post: - summary: tracking table source + description: Success + schema: + $ref: '#/definitions/ResponseComponentDownload' + operationId: post_download_data + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqComponentDownload' tags: - - table - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - table_name - - namespace - properties: - table_name: - type: string - example: breast_hetero_guest - namespace: - type: string - example: experiment + - data + /data/component/upload: + post: responses: '200': - description: tracking success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: array - items: - type: object - example: - { - "parent_table_name": xxx, - "parent_table_namespace": xxx, - "source_table_name": xxx, - "source_table_namespace": xxx - } - '404': - description: tracking failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no find table - - '/table/tracking/job': - post: - summary: tracking using table job + description: Success + schema: + $ref: '#/definitions/ResponseUploadData' + operationId: post_upload_data + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqComponentUpload' tags: - - table - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - table_name - - namespace - properties: - table_name: - type: string - example: breast_hetero_guest - namespace: - type: string - example: experiment + - data + /data/component/upload/file: + post: responses: '200': - description: tracking success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: array - items: - type: object - example: - { - "count": 5, - "job": - [ - "202104212104472450460", - "202104212127150470680", - "202104220937051579910", - "202104212038599210200", - "202104212131462630720" - ] - } - '404': - description: tracking failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no find table - - '/table/preview': - post: - summary: table data preview + description: Success + schema: + $ref: '#/definitions/ResponseUploadData' + operationId: post_upload_data_file + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqComponentUploadFile' tags: - - table - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - name - - namespace - properties: - name: - type: string - example: "guest" - namespace: - type: string - example: "data" + - data + /data/download: + get: responses: '200': - description: get preview table success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - '404': - description: no found table - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 210 - retmsg: - type: string - example: no find table - - '/job/submit': - post: - summary: submit job + description: Success + schema: + $ref: '#/definitions/ResponseUploadData' + operationId: get_data_Download + parameters: + - required: true + description: Name of the data table + in: query + name: name + type: string + - required: true + description: Namespace of the data table + in: query + name: namespace + type: string + - required: false + description: Whether the first row of the file is the data's head + in: query + name: header + type: string tags: - - job - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - dsl - - runtime_conf - properties: - dsl: - type: object - example: - { - "components": - { - "dataio_0": - { - "input": - { - "data": - { - "data": - [ - "reader_0.table" - ] - } - }, - "module": "DataIO", - "need_deploy": true, - "output": - { - "data": - [ - "train" - ], - "model": - [ - "dataio" - ] - } - }, - "evaluation_0": - { - "input": - { - "data": - { - "data": - [ - "hetero_lr_0.train" - ] - } - }, - "module": "Evaluation", - "output": - { - "data": - [ - "evaluate" - ] - } - }, - "hetero_feature_binning_0": - { - "input": - { - "data": - { - "data": - [ - "intersection_0.train" - ] - } - }, - "module": "HeteroFeatureBinning", - "output": - { - "data": - [ - "train" - ], - "model": - [ - "hetero_feature_binning" - ] - } - }, - "hetero_feature_selection_0": - { - "input": - { - "data": - { - "data": - [ - "hetero_feature_binning_0.train" - ] - }, - "isometric_model": - [ - "hetero_feature_binning_0.hetero_feature_binning" - ] - }, - "module": "HeteroFeatureSelection", - "output": - { - "data": - [ - "train" - ], - "model": - [ - "selected" - ] - } - }, - "hetero_lr_0": - { - "input": - { - "data": - { - "train_data": - [ - "hetero_feature_selection_0.train" - ] - } - }, - "module": "HeteroLR", - "output": - { - "data": - [ - "train" - ], - "model": - [ - "hetero_lr" - ] - } - }, - "intersection_0": - { - "input": - { - "data": - { - "data": - [ - "dataio_0.train" - ] - } - }, - "module": "Intersection", - "output": - { - "data": - [ - "train" - ] - } - }, - "reader_0": - { - "module": "Reader", - "output": - { - "data": - [ - "table" - ] - } - } - } - } - runtime_conf: - type: object - example: - { - "component_parameters": - { - "common": - { - "hetero_lr_0": - { - "alpha": 0.01, - "batch_size": 320, - "init_param": - { - "init_method": "random_uniform" - }, - "learning_rate": 0.15, - "max_iter": 3, - "optimizer": "rmsprop", - "penalty": "L2" - }, - "intersection_0": - { - "intersect_method": "raw", - "only_output_key": false, - "sync_intersect_ids": true - } - }, - "role": - { - "guest": - { - "0": - { - "dataio_0": - { - "label_name": "y", - "label_type": "int", - "output_format": "dense", - "with_label": true - }, - "reader_0": - { - "table": - { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - } - } - }, - "host": - { - "0": - { - "dataio_0": - { - "output_format": "dense", - "with_label": false - }, - "evaluation_0": - { - "need_run": false - }, - "reader_0": - { - "table": - { - "name": "breast_hetero_host", - "namespace": "experiment" - } - } - } - } - } - }, - "dsl_version": "2", - "initiator": - { - "party_id": 9999, - "role": "guest" - }, - "job_parameters": - { - "common": - { - "auto_retries": 1, - "computing_partitions": 8, - "task_cores": 4, - "task_parallelism": 2 - } - }, - "role": - { - "arbiter": - [ - 10000 - ], - "guest": - [ - 9999 - ], - "host": - [ - 10000 - ] - } - } + - data + /job/clean: + post: responses: '200': - description: submit job success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "board_url": xxx, - "code": 0, - "dsl_path": xxx, - "job_id": xxx, - "logs_directory": xxx, - "message": "success", - "model_info": - { - "model_id": xxx, - "model_version": xxx - }, - "pipeline_dsl_path": xxx, - "runtime_conf_on_party_path": xxx, - "runtime_conf_path": xxx, - "train_runtime_conf_path": xxx - } - '404': - description: submit job failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: "config error" - - '/job/stop': - post: - summary: stop job + description: Success + schema: + $ref: '#/definitions/ResponseJobClean' + operationId: post_clean_job + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqJobId' tags: - job - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - job_id - properties: - job_id: - type: string - example: 202103231958539401540 - stop_status: - type: string - default: cancel - example: "failed" - description: "failed or cancel" + /job/dag/dependency: + get: responses: '200': - description: stop job success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - '404': - description: stop job failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no find job - - '/job/rerun': - post: - summary: rerun job + description: Success + schema: + $ref: '#/definitions/CommonDictModel' + operationId: get_job_DagDependency + parameters: + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string tags: - job - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - job_id - properties: - job_id: - type: string - example: 202103231958539401540 + /job/data/view: + get: responses: '200': - description: rerun job success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - '404': - description: rerun job failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no find job - - '/job/query': - post: - summary: query job + description: Success + schema: + $ref: '#/definitions/ResponseJobClean' + operationId: get_data_view + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqDataView' tags: - job - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - job_id - properties: - job_id: - type: string + /job/list/query: + get: responses: '200': - description: query job success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: array - items: - type: object - example: - { - "f_apply_resource_time": xxx, - "f_cancel_signal": false, - "f_cancel_time": xxx, - "f_cores": 8, - "f_create_date": xxx, - "f_create_time": xxx, - "f_description": "", - "f_dsl": {}, - "f_elapsed": 14380, - "f_end_date": xxx, - "f_end_scheduling_updates": 1, - "f_end_time": xxx, - "f_engine_name": "EGGROLL", - "f_engine_type": "computing", - "f_initiator_party_id": "20001", - "f_initiator_role": "guest", - "f_is_initiator": true, - "f_job_id": xxx, - "f_memory": 0, - "f_name": "", - "f_party_id": "20001", - "f_progress": 14, - "f_ready_signal": false, - "f_ready_time": null, - "f_remaining_cores": 8, - "f_remaining_memory": 0, - "f_rerun_signal": false, - "f_resource_in_use": false, - "f_return_resource_time": xxx, - "f_role": "guest", - "f_roles": {}, - "f_runtime_conf": {}, - "f_runtime_conf_on_party": {}, - "f_start_date": xxx, - "f_start_time": xxx, - "f_status": "failed", - "f_status_code": null, - "f_tag": "job_end", - "f_train_runtime_conf": {}, - "f_update_date": xxx, - "f_update_time": xxx, - "f_user": {}, - "f_user_id": "" - } - '404': - description: query job failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no find job - - '/job/list/job': - post: - summary: list jobs + description: Success + schema: + $ref: '#/definitions/CommonDictModel' + operationId: get_job_QueryJobList + parameters: + - required: false + description: Limit of rows or entries + in: query + name: limit + type: string + - required: false + description: Page number + in: query + name: page + type: string + - required: false + description: Job ID + in: query + name: job_id + type: string + - required: false + description: Description information + in: query + name: description + type: string + - required: false + description: Participant information + in: query + name: partner + type: string + - required: false + description: Site ID + in: query + name: party_id + type: string + - required: false + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: false + description: Status of the job or task + in: query + name: status + type: string + - required: false + description: Field name for sorting + in: query + name: order_by + type: string + - required: false + description: 'Sorting order: asc/desc' + in: query + name: order + type: string + - required: false + description: Username provided by the upper-level system + in: header + name: user_name + type: string tags: - job - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - limit: - type: integer - description: '`0` means no limit' - example: 20 - page: - type: integer - example: 1 - job_id: - type: string - example: '202112020129140220090' - party_id: - type: integer - example: 9999 - role: - type: array - items: - type: string - enum: - - guest - - host - - arbiter - - local - status: - type: array - items: - type: string - enum: - - success - - running - - waiting - - failed - - canceled - description: - type: string - description: '`notes` on front-end' - order_by: - type: string - description: 'defaults `create_time`' - enum: - - create_time - - start_time - - end_time - - elapsed - order: - type: string - description: 'defaults `desc`' - enum: - - asc - - desc + /job/log/download: + post: responses: '200': - description: jobs list - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - properties: - count: - type: integer - example: 1 - jobs: - type: array - items: - type: object - example: - apply_resource_time: 1638379762883 - cancel_signal: false - cancel_time: - cores: 4 - create_date: '2021-12-02 01:29:18' - create_time: 1638379758581 - description: '' - dsl: - components: - dataio_0: - input: - data: - data: - - reader_0.data - module: DataIO - output: - data: - - data - model: - - model - hetero_feature_binning_0: - input: - data: - data: - - intersection_0.data - module: HeteroFeatureBinning - output: - data: - - data - model: - - model - intersection_0: - input: - data: - data: - - dataio_0.data - module: Intersection - output: - cache: - - cache - data: - - data - reader_0: - module: Reader - output: - data: - - data - provider: fate_flow@1.7.0 - elapsed: 116548 - end_date: '2021-12-02 01:31:19' - end_scheduling_updates: 1 - end_time: 1638379879547 - engine_name: STANDALONE - engine_type: computing - initiator_party_id: '10000' - initiator_role: guest - is_initiator: true - job_id: '202112020129140220090' - memory: 0 - name: '' - partners: - - 9999 - party_id: 10000 - progress: 100 - ready_signal: false - ready_time: - remaining_cores: 4 - remaining_memory: 0 - rerun_signal: false - resource_in_use: false - return_resource_time: 1638379879568 - role: guest - roles: - guest: - - 10000 - host: - - 9999 - runtime_conf: - component_parameters: - common: - hetero_feature_binning_0: - adjustment_factor: 0.5 - bin_indexes: -1 - bin_names: - bin_num: 10 - category_indexes: - category_names: - compress_thres: 10000 - error: 0.001 - head_size: 10000 - local_only: false - method: quantile - transform_param: - transform_cols: - - 0 - - 1 - - 2 - transform_names: - transform_type: woe - role: - guest: - '0': - dataio_0: - with_label: true - reader_0: - table: - name: breast_hetero_guest - namespace: experiment - host: - '0': - dataio_0: - with_label: false - hetero_feature_binning_0: - transform_param: - transform_type: - reader_0: - table: - name: breast_hetero_host - namespace: experiment - conf_path: "/tmp/tmp2zc5tf8b/job_runtime_conf.json" - dsl_path: "/tmp/tmp2zc5tf8b/job_dsl.json" - dsl_version: 2 - initiator: - party_id: 10000 - role: guest - job_parameters: - common: - adaptation_parameters: - if_initiator_baseline: true - request_task_cores: 4 - task_cores_per_node: 4 - task_memory_per_node: 0 - task_nodes: 1 - auto_retries: 0 - auto_retry_delay: 1 - computing_engine: STANDALONE - computing_partitions: 4 - eggroll_run: { } - engines_address: { } - federated_mode: SINGLE - federated_status_collect_type: PUSH - federation_engine: STANDALONE - job_type: train - model_id: guest-10000#host-9999#model - model_version: '202112020129140220090' - pulsar_run: { } - rabbitmq_run: { } - spark_run: { } - storage_engine: STANDALONE - task_parallelism: 1 - role: - guest: - - 10000 - host: - - 9999 - runtime_conf_on_party: - component_parameters: - common: - hetero_feature_binning_0: - adjustment_factor: 0.5 - bin_indexes: -1 - bin_names: - bin_num: 10 - category_indexes: - category_names: - compress_thres: 10000 - error: 0.001 - head_size: 10000 - local_only: false - method: quantile - transform_param: - transform_cols: - - 0 - - 1 - - 2 - transform_names: - transform_type: woe - role: - guest: - '0': - dataio_0: - with_label: true - reader_0: - table: - name: breast_hetero_guest - namespace: experiment - host: - '0': - dataio_0: - with_label: false - hetero_feature_binning_0: - transform_param: - transform_type: - reader_0: - table: - name: breast_hetero_host - namespace: experiment - conf_path: "/tmp/tmp2zc5tf8b/job_runtime_conf.json" - dsl_path: "/tmp/tmp2zc5tf8b/job_dsl.json" - dsl_version: 2 - initiator: - party_id: 10000 - role: guest - job_parameters: - adaptation_parameters: - if_initiator_baseline: false - request_task_cores: 4 - task_cores_per_node: 4 - task_memory_per_node: 0 - task_nodes: 1 - auto_retries: 0 - auto_retry_delay: 1 - computing_engine: STANDALONE - computing_partitions: 4 - eggroll_run: - eggroll.session.processors.per.node: 4 - engines_address: - computing: - cores_per_node: 20 - nodes: 1 - federation: - cores_per_node: 20 - nodes: 1 - storage: - cores_per_node: 20 - nodes: 1 - federated_mode: SINGLE - federated_status_collect_type: PUSH - federation_engine: STANDALONE - job_type: train - model_id: guest-10000#host-9999#model - model_version: '202112020129140220090' - pulsar_run: { } - rabbitmq_run: { } - spark_run: { } - storage_engine: STANDALONE - task_parallelism: 1 - role: - guest: - - 10000 - host: - - 9999 - start_date: '2021-12-02 01:29:22' - start_time: 1638379762999 - status: success - status_code: - tag: job_end - train_runtime_conf: { } - update_date: '2021-12-02 01:32:04' - update_time: 1638379924749 - user: - guest: - '10000': '' - host: - '9999': '' - user_id: '' - - '/job/list/task': - post: - summary: list tasks + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_download_job_logs + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqJobId' + produces: + - application/gzip tags: - job - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - limit: - type: integer - description: '`0` means no limit' - example: 20 - page: - type: integer - example: 1 - job_id: - type: string - example: '202112020129140220090' - party_id: - type: integer - example: 9999 - role: - type: string - enum: - - guest - - host - - arbiter - - local - component_name: - type: string - example: upload_0 - order_by: - type: string - description: 'defaults `create_time`' - enum: - - create_time - - start_time - - end_time - - elapsed - order: - type: string - description: 'defaults `asc`' - enum: - - asc - - desc + /job/notes/add: + post: responses: '200': - description: jobs list - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - properties: - count: - type: integer - example: 1 - tasks: - type: array - items: - type: object - example: - auto_retries: 0 - auto_retry_delay: 1 - cmd: - - "/root/Codes/FATE/venv/bin/python3.6" - - "/root/Codes/FATE/fateflow/python/fate_flow/worker/task_executor.py" - - "--job_id" - - '202112020129140220090' - - "--component_name" - - hetero_feature_binning_0 - - "--task_id" - - 202112020129140220090_hetero_feature_binning_0 - - "--task_version" - - 0 - - "--role" - - guest - - "--party_id" - - '10000' - - "--config" - - "/root/Codes/FATE/fateflow/jobs/202112020129140220090/guest/10000/hetero_feature_binning_0/202112020129140220090_hetero_feature_binning_0/0/task_executor/6b6f4b1852cc11ec8a8700155d13c16c/config.json" - - "--result" - - "/root/Codes/FATE/fateflow/jobs/202112020129140220090/guest/10000/hetero_feature_binning_0/202112020129140220090_hetero_feature_binning_0/0/task_executor/6b6f4b1852cc11ec8a8700155d13c16c/result.json" - - "--log_dir" - - "/root/Codes/FATE/fateflow/logs/202112020129140220090/guest/10000/hetero_feature_binning_0" - - "--parent_log_dir" - - "/root/Codes/FATE/fateflow/logs/202112020129140220090/guest/10000" - - "--worker_id" - - 6b6f4b1852cc11ec8a8700155d13c16c - - "--run_ip" - - 127.0.0.1 - - "--job_server" - - 127.0.0.1:9380 - - "--session_id" - - 202112020129140220090_hetero_feature_binning_0_0_guest_10000 - - "--federation_session_id" - - 202112020129140220090_hetero_feature_binning_0_0 - component_module: HeteroFeatureBinning - component_name: hetero_feature_binning_0 - component_parameters: - CodePath: HeteroFeatureBinningGuest - ComponentParam: - _feeded_deprecated_params: [ ] - _is_raw_conf: false - _name: HeteroFeatureBinning#hetero_feature_binning_0 - _user_feeded_params: - - head_size - - category_names - - bin_num - - transform_param.transform_names - - transform_param - - compress_thres - - error - - method - - bin_indexes - - transform_param.transform_type - - bin_names - - category_indexes - - local_only - - transform_param.transform_cols - - adjustment_factor - adjustment_factor: 0.5 - bin_indexes: -1 - bin_names: - bin_num: 10 - category_indexes: - category_names: - compress_thres: 10000 - encrypt_param: - key_length: 1024 - method: Paillier - error: 0.001 - head_size: 10000 - local_only: false - method: quantile - need_run: true - optimal_binning_param: - adjustment_factor: - init_bin_nums: 1000 - init_bucket_method: quantile - max_bin: - max_bin_pct: 1 - metric_method: iv - min_bin_pct: 0.05 - mixture: true - skip_static: false - transform_param: - transform_cols: - - 0 - - 1 - - 2 - transform_names: - transform_type: woe - conf_path: "/tmp/tmp2zc5tf8b/job_runtime_conf.json" - dsl_path: "/tmp/tmp2zc5tf8b/job_dsl.json" - dsl_version: 2 - initiator: - party_id: 10000 - role: guest - job_parameters: - common: - adaptation_parameters: - if_initiator_baseline: true - request_task_cores: 4 - task_cores_per_node: 4 - task_memory_per_node: 0 - task_nodes: 1 - auto_retries: 0 - auto_retry_delay: 1 - computing_engine: STANDALONE - computing_partitions: 4 - eggroll_run: { } - engines_address: { } - federated_mode: SINGLE - federated_status_collect_type: PUSH - federation_engine: STANDALONE - job_type: train - model_id: guest-10000#host-9999#model - model_version: '202112020129140220090' - pulsar_run: { } - rabbitmq_run: { } - spark_run: { } - storage_engine: STANDALONE - task_parallelism: 1 - local: - party_id: 10000 - role: guest - module: HeteroFeatureBinning - role: - guest: - - 10000 - host: - - 9999 - create_date: '2021-12-02 01:29:21' - create_time: 1638379761918 - elapsed: 9095 - end_date: '2021-12-02 01:31:04' - end_time: 1638379864051 - engine_conf: - computing_engine: STANDALONE - federated_mode: SINGLE - federated_status_collect_type: PUSH - initiator_party_id: '10000' - initiator_role: guest - job_id: '202112020129140220090' - party_id: '10000' - party_status: success - provider_info: - class_path: - feature_instance: feature.instance.Instance - feature_vector: feature.sparse_vector.SparseVector - homo_model_convert: protobuf.homo_model_convert.homo_model_convert - interface: components.components.Components - model: protobuf.generated - model_migrate: protobuf.model_migrate.model_migrate - env: - PYTHONPATH: "/root/Codes/FATE/python" - name: fate - path: "/root/Codes/FATE/python/federatedml" - version: 1.7.0 - role: guest - run_ip: 127.0.0.1 - run_on_this_party: true - run_pid: 29934 - start_date: '2021-12-02 01:30:47' - start_time: 1638379847118 - status: success - status_code: - task_id: 202112020129140220090_hetero_feature_binning_0 - task_version: 0 - update_date: '2021-12-02 01:31:06' - update_time: 1638379866439 - worker_id: 6b6f4b1852cc11ec8a8700155d13c16c - - '/job/update': - post: - summary: job notes + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_add_notes + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqNodesAdd' tags: - job - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - job_id - - role - - party_id - - notes - properties: - job_id: - type: string - example: "2022xxx" - role: - type: string - example: guest - party_id: - type: integer - example: 10000 - notes: - type: string - example: this is a test + /job/query: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no find job - - '/job/parameter/update': - post: - summary: update job parameter + description: Success + schema: + $ref: '#/definitions/CommonListToDictModel' + operationId: get_job_QueryJob + parameters: + - required: false + description: Job ID + in: query + name: job_id + type: string + - required: false + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: false + description: Site ID + in: query + name: party_id + type: string + - required: false + description: Status of the job or task + in: query + name: status + type: string + - required: false + description: Username provided by the upper-level system + in: header + name: user_name + type: string tags: - job - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - job_id - properties: - job_id: - type: string - example: "2022xxx" - component_parameters: - type: object - example: - { - "common": - { - "hetero_lr_0": - { - "max_iter": 10 - } - } - } - job_parameters: - type: object - example: - { - "common": - { - "auto_retries": 2 - } - } + /job/queue/clean: + post: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "component_parameters": - { - "common": - { - "hetero_lr_0": - { - "alpha": 0.01, - "batch_size": 320, - "init_param": - { - "init_method": "random_uniform" - }, - "learning_rate": 0.15, - "max_iter": 10, - "optimizer": "rmsprop", - "penalty": "L2" - }, - "intersection_0": - { - "intersect_method": "raw", - "only_output_key": false, - "sync_intersect_ids": true - } - }, - "role": - { - "guest": - { - "0": - { - "dataio_0": - { - "label_name": "y", - "label_type": "int", - "output_format": "dense", - "with_label": true - }, - "reader_0": - { - "table": - { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - } - } - }, - "host": - { - "0": - { - "dataio_0": - { - "output_format": "dense", - "with_label": false - }, - "evaluation_0": - { - "need_run": false - }, - "reader_0": - { - "table": - { - "name": "breast_hetero_host", - "namespace": "experiment" - } - } - } - } - } - }, - "components": - [], - "job_parameters": - { - "common": - { - "adaptation_parameters": - { - "if_initiator_baseline": true, - "request_task_cores": 4, - "task_cores_per_node": 4, - "task_memory_per_node": 0, - "task_nodes": 1 - }, - "auto_retries": 2, - "auto_retry_delay": 1, - "computing_engine": "EGGROLL", - "computing_partitions": 4, - "eggroll_run": - {}, - "engines_address": - {}, - "federated_mode": "MULTIPLE", - "federated_status_collect_type": "PUSH", - "inheritance_info": - {}, - "job_type": "train", - "model_id": "arbiter-10001#guest-20001#host-10001#model", - "model_version": "202204251958539401540", - "pulsar_run": - {}, - "rabbitmq_run": - {}, - "spark_run": - {}, - "task_parallelism": 1 - } - }, - "src_party_id": "20001", - "src_role": "guest" - } - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no find job - - '/job/config': - post: - summary: job config + description: Success + schema: + $ref: '#/definitions/ResponseJobClean' + operationId: post_clean_queue tags: - job - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - job_id - properties: - job_id: - type: string - example: "2022xxx" - role: - type: string - example: guest - party_id: - type: integer - example: 10000 + /job/rerun: + post: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "dsl": - {}, - "job_id": "2022xxx", - "model_info": - {}, - "runtime_conf": - {}, - "train_runtime_conf": - {} - } - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no find job - - '/job/log/download': - post: - summary: download job log (tar.gz) + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_request_rerun_job + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqJobId' tags: - job - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - job_id - properties: - job_id: - type: string + /job/stop: + post: responses: '200': - description: get job log success - content: - application/octet-stream: - schema: - type: string - description: file xxx_log.tar.gz - '404': - description: get job list failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 404 - retmsg: - type: string - example: Log file path xxx not found. Please check if the job id is valid. - - '/job/log/path': - post: - summary: job log path + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_request_stop_job + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqJobId' tags: - job - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - job_id - properties: - job_id: - type: string + /job/submit: + post: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "logs_directory": "/data/projects/fate/fateflow/logs/xxx" - } - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no find job - - '/job/task/query': - post: - summary: query task + description: Success + schema: + $ref: '#/definitions/ResponseJobSubmit' + operationId: post_submit_job + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqSubmitJob' tags: - job - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - job_id - properties: - job_id: - type: string - role: - type: string - example: guest - party_id: - type: integer - example: 10000 - component_name: - type: string - example: reader_0 + /job/task/list/query: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: array - description: tasks list - items: - type: object - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no find task - - '/job/clean': - post: - summary: clean job + description: Success + schema: + $ref: '#/definitions/CommonDictModel' + operationId: get_job_QueryTaskList + parameters: + - required: false + description: Limit of rows or entries + in: query + name: limit + type: string + - required: false + description: Page number + in: query + name: page + type: string + - required: false + description: Job ID + in: query + name: job_id + type: string + - required: false + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: false + description: Site ID + in: query + name: party_id + type: string + - required: false + description: Task name + in: query + name: task_name + type: string + - required: false + description: Field name for sorting + in: query + name: order_by + type: string + - required: false + description: 'Sorting order: asc/desc' + in: query + name: order + type: string tags: - job - requestBody: - required: true - content: - application/json: - schema: - required: - - job_id - type: object - properties: - job_id: - type: string - role: - type: string - example: guest - party_id: - type: integer - example: 10000 - component_name: - type: string - example: reader_0 + /job/task/query: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no find task - - '/job/clean/queue': - post: - summary: cancel waiting job + description: Success + schema: + $ref: '#/definitions/CommonListToDictModel' + operationId: get_job_QueryTask + parameters: + - required: false + description: Job ID + in: query + name: job_id + type: string + - required: false + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: false + description: Site ID + in: query + name: party_id + type: string + - required: false + description: Status of the job or task + in: query + name: status + type: string + - required: false + description: Task name + in: query + name: task_name + type: string + - required: false + description: Task ID + in: query + name: task_id + type: string + - required: false + description: Task version + in: query + name: task_version + type: string tags: - job + /log/count: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "202204261616186991350": 0, - "202204261616198643190": 0, - "202204261616210073410": 0 - } - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: server error - - '/tracking/job/data_view': - post: - summary: data view + description: Success + schema: + $ref: '#/definitions/CommonDataModel' + operationId: get_log_Count + parameters: + - required: true + description: Log level or type + in: query + name: log_type + type: string + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: false + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: false + description: Site ID + in: query + name: party_id + type: string + - required: false + description: Task name + in: query + name: task_name + type: string + - required: false + description: Instance ID of the FATE Flow service + in: query + name: instance_id + type: string tags: - - tracking - requestBody: - required: true - content: - application/json: - schema: - required: - - job_id - - role - - party_id - type: object - properties: - job_id: - type: string - role: - type: string - example: guest - party_id: - type: integer - example: 10000 + - log + /log/query: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "dataset": - { - "guest": - { - "9999": - { - "Reader_0": "xxx.xxx" - } - }, - "host": - { - "10000": - { - "Reader_0": "xxx.xxx" - } - } - }, - "model_summary": - {}, - "partner": - { - "host": - [ - 10000 - ] - }, - "roles": - { - "guest": - [ - 9999 - ], - "host": - [ - 10000 - ] - } - } - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: error - - '/tracking/component/metric/all': + description: Success + schema: + $ref: '#/definitions/CommonListDictModel' + operationId: get_log_Get + parameters: + - required: true + description: Log level or type + in: query + name: log_type + type: string + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: false + description: Task name + in: query + name: task_name + type: string + - required: false + description: Starting line number + in: query + name: begin + type: string + - required: false + description: Ending line number + in: query + name: end + type: string + - required: false + description: Instance ID of the FATE Flow service + in: query + name: instance_id + type: string + tags: + - log + /model/delete: post: - summary: get component all metric + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ModelDelete' + operationId: post_delete_model + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqModelDelete' tags: - - tracking - requestBody: - required: true - content: - application/json: - schema: - required: - - job_id - - role - - party_id - - component_name - type: object - properties: - job_id: - type: string - role: - type: string - example: guest - party_id: - type: integer - example: 10000 - component_name: - type: string - example: HeteroSecureBoost_0 + - model + /model/export: + post: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "train": - { - "loss": - { - "data": - [ - [ - 0, - 0.6076415445876732 - ], - [ - 1, - 0.5374539452565573 - ], - [ - 2, - 0.4778598986135903 - ], - [ - 3, - 0.42733599866560723 - ], - [ - 4, - 0.38433409799127843 - ] - ], - "meta": - { - "Best": 0.38433409799127843, - "curve_name": "loss", - "metric_type": "LOSS", - "name": "train", - "unit_name": "iters" - } - } - } - } - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: error - - '/tracking/component/metrics': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_export + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqModelExport' + tags: + - model + /model/import: post: - summary: get component metric name and namespace + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_import_model + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqModelImport' tags: - - tracking - requestBody: - required: true - content: - application/json: - schema: - required: - - job_id - - role - - party_id - - component_name - type: object - properties: - job_id: - type: string - role: - type: string - example: guest - party_id: - type: integer - example: 10000 - component_name: - type: string - example: Intersection_0 + - model + /model/load: + post: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "train": - [ - "intersection" - ] - } - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: error - - '/tracking/component/metric_data': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_load + tags: + - model + /model/migrate: post: - summary: get component metric data + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_migrate tags: - - tracking - requestBody: - required: true - content: - application/json: - schema: - required: - - job_id - - role - - party_id - - component_name - - metric_name - - metric_namespace - type: object - properties: - job_id: - type: string - role: - type: string - example: guest - party_id: - type: integer - example: 10000 - component_name: - type: string - example: Intersection_0 - metric_name: - type: string - example: intersection - metric_namespace: - type: string - example: train + - model + /model/restore: + post: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "data": - [ - [ - "intersect_count", - 569 - ], - [ - "intersect_rate", - 1.0 - ], - [ - "unmatched_count", - 0 - ], - [ - "unmatched_rate", - 0.0 - ] - ], - "meta": - { - "intersect_method": "raw", - "join_method": "inner_join", - "metric_type": "INTERSECTION", - "name": "intersection" - } - } - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: error - - '/tracking/component/parameters': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_restore + tags: + - model + /model/store: post: - summary: get component parameters + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_store tags: - - tracking - requestBody: - required: true - content: - application/json: - schema: - required: - - job_id - - role - - party_id - - component_name - type: object - properties: - job_id: - type: string - role: - type: string - example: guest - party_id: - type: integer - example: 10000 - component_name: - type: string - example: Intersection_0 + - model + /output/data/display: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "ComponentParam": - { - "_feeded_deprecated_params": - [ - "repeated_id_owner", - "intersect_cache_param", - "join_role", - "encode_params", - "allow_info_share", - "repeated_id_process", - "random_bit", - "with_encode", - "info_owner" - ], - "_is_raw_conf": false, - "_name": "Intersection#Intersection_0", - "_user_feeded_params": - [ - "repeated_id_owner", - "encode_params.salt", - "intersect_method", - "intersect_cache_param.encrypt_type", - "encode_params.encode_method", - "encode_params", - "intersect_cache_param", - "intersect_cache_param.id_type", - "intersect_cache_param.use_cache", - "join_role", - "allow_info_share", - "repeated_id_process", - "sync_intersect_ids", - "random_bit", - "only_output_key", - "with_encode", - "encode_params.base64", - "info_owner" - ], - "allow_info_share": false, - "cardinality_only": false, - "dh_params": - { - "hash_method": "sha256", - "key_length": 1024, - "salt": "" - }, - "encode_params": - { - "base64": false, - "encode_method": "none", - "salt": "" - }, - "info_owner": "guest", - "intersect_cache_param": - { - "encrypt_type": "sha256", - "id_type": "phone", - "use_cache": false - }, - "intersect_method": "raw", - "intersect_preprocess_params": - { - "encrypt_method": "rsa", - "false_positive_rate": 0.001, - "filter_owner": "guest", - "hash_method": "sha256", - "preprocess_method": "sha256", - "preprocess_salt": "", - "random_state": null - }, - "join_method": "inner_join", - "join_role": "guest", - "new_sample_id": false, - "only_output_key": false, - "random_bit": 128, - "raw_params": - { - "base64": false, - "hash_method": "none", - "join_role": "guest", - "salt": "", - "use_hash": false - }, - "repeated_id_owner": "guest", - "repeated_id_process": false, - "rsa_params": - { - "final_hash_method": "sha256", - "hash_method": "sha256", - "key_length": 1024, - "random_base_fraction": null, - "random_bit": 128, - "salt": "", - "split_calculation": false - }, - "run_cache": false, - "run_preprocess": false, - "sample_id_generator": "guest", - "sync_cardinality": false, - "sync_intersect_ids": true, - "with_encode": false, - "with_sample_id": false - }, - "module": "Intersection" - } - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: error - - '/tracking/component/summary/download': - post: - summary: get component summary + description: Success + schema: + $ref: '#/definitions/ResponseTableDisplay' + operationId: get_output_OutputDataDisplay + parameters: + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: true + description: Task name + in: query + name: task_name + type: string tags: - - tracking - requestBody: - required: true - content: - application/json: - schema: - required: - - job_id - - role - - party_id - - component_name - type: object - properties: - job_id: - type: string - role: - type: string - example: guest - party_id: - type: integer - example: 10000 - component_name: - type: string - example: Intersection_0 + - output + /output/data/download: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "cardinality_only": false, - "intersect_num": 569, - "intersect_rate": 1.0 - } - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: error - - '/tracking/component/output/model': - post: - summary: get component output model + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: get_output_OutputDataDownload + parameters: + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: true + description: Task name + in: query + name: task_name + type: string + - required: false + description: Primary key for output data or model of the task + in: query + name: output_key + type: string + produces: + - application/gzip tags: - - tracking - requestBody: - required: true - content: - application/json: - schema: - required: - - job_id - - role - - party_id - - component_name - type: object - properties: - job_id: - type: string - role: - type: string - example: guest - party_id: - type: integer - example: 10000 - component_name: - type: string - example: Intersection_0 + - output + /output/data/table: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: error - - '/tracking/component/output/data': + description: Success + schema: + $ref: '#/definitions/ResponseDataTable' + operationId: get_output_OutputDataTable + parameters: + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: true + description: Task name + in: query + name: task_name + type: string + tags: + - output + /output/metric/delete: post: - summary: get component output data + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseDeleteMetric' + operationId: post_delete_metric + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqMetricDelete' tags: - - tracking - requestBody: - required: true - content: - application/json: - schema: - required: - - job_id - - role - - party_id - - component_name - type: object - properties: - job_id: - type: string - role: - type: string - example: guest - party_id: - type: integer - example: 10000 - component_name: - type: string - example: Intersection_0 + - output + /output/metric/key/query: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - [ - [ - [ - "58", - 1, - -0.422281, - -0.558102, - -0.506991, - -0.450875, - -1.326851, - -1.223647, - -1.296979, - -1.575895, - -0.747019, - -1.166825 - ], - [ - "66", - 1, - -1.213336, - 0.957974, - -1.19832, - -0.966647, - 0.983301, - -0.558944, - -0.854288, - -0.752745, - -0.036814, - 0.452425 - ] - ] - ] - meta: - type: object - example: - { - "header": - [ - [ - "id", - "y", - "x0", - "x1", - "x2", - "x3", - "x4", - "x5", - "x6", - "x7", - "x8", - "x9" - ] - ], - "names": - [ - "data_0" - ], - "total": - [ - 569 - ] - } - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: error - - '/tracking/component/output/data/download': - post: - summary: download component output data (tar.gz) + description: Success + schema: + $ref: '#/definitions/OutputQuery' + operationId: get_output_QueryMetricKey + parameters: + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: true + description: Task name + in: query + name: task_name + type: string tags: - - tracking - requestBody: - required: true - content: - application/json: - schema: - required: - - job_id - - role - - party_id - - component_name - type: object - properties: - job_id: - type: string - role: - type: string - example: guest - party_id: - type: integer - example: 10000 - component_name: - type: string - example: Intersection_0 + - output + /output/metric/query: + get: responses: '200': - description: success - content: - application/octet-stream: - schema: - type: string - example: "" - description: file data.tar.gz - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: error - - '/tracking/component/output/data/table': + description: Success + schema: + $ref: '#/definitions/CommonDictModel' + operationId: get_output_QueryMetric + parameters: + - required: true + description: Site ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: true + description: Task name + in: query + name: task_name + type: string + - required: false + description: Filter conditions + in: query + name: filters + type: string + tags: + - output + /output/model/delete: post: - summary: get component output data table info + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_output_delete_model + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqMetricDelete' tags: - - tracking - requestBody: - required: true - content: - application/json: - schema: - required: - - job_id - - role - - party_id - - component_name - type: object - properties: - job_id: - type: string - role: - type: string - example: guest - party_id: - type: integer - example: 10000 - component_name: - type: string - example: Intersection_0 + - output + /output/model/download: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: array - items: - type: object - example: - [ - { - "data_name": "data_0", - "table_name": "ab035e12b09711ec943e525400c367ed", - "table_namespace": "output_data_202203311009181495690_Intersection_0_0" - } - ] - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: No found table, please check if the parameters are correct - - '/tracking/component/list': - post: - summary: get component list + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: get_output_Download + parameters: + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: true + description: Task name + in: query + name: task_name + type: string + produces: + - application/x-tar tags: - - tracking - requestBody: - required: true - content: - application/json: - schema: - required: - - job_id - type: object - properties: - job_id: - type: string + - output + /output/model/query: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: - { - "components": - [ - "HeteroFeatureSelection_0", - "DataIO_0", - "Reader_0", - "HeteroSecureBoost_0", - "HeteroFeatureBinning_0", - "Intersection_0", - "Evaluation_0" - ] - } - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: error - - '/resource/query': - post: - summary: query conputing engine resource + description: Success + schema: + $ref: '#/definitions/ResponseFateFlow' + operationId: get_output_QueryModel + parameters: + - required: true + description: Job ID + in: query + name: job_id + type: string + - required: true + description: 'Role of the participant: guest/host/arbiter/local' + in: query + name: role + type: string + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: true + description: Task name + in: query + name: task_name + type: string tags: - - resource - requestBody: - required: true - content: - application/json: - schema: - required: - - engine_name - type: object - properties: - engine_name: - type: string - example: EGGROLL, SPARK + - output + /permission/delete: + post: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: {"computing_engine_resource":{"f_cores":16,"f_create_date":"2022-02-10 16:14:03","f_create_time":1644480843818,"f_engine_config":{"cores_per_node":16,"nodes":1},"f_engine_entrance":"fate_on_eggroll","f_engine_name":"EGGROLL","f_engine_type":"computing","f_memory":0,"f_nodes":1,"f_remaining_cores":16,"f_remaining_memory":0,"f_update_date":"2022-04-27 15:48:33","f_update_time":1651045713996},"use_resource_job":[]} - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: error - - '/resource/return': + description: Success + schema: + $ref: '#/definitions/ResponseGrantPermission' + operationId: post_delete + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqGrant' + tags: + - permission + /permission/grant: post: - summary: return job resource + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseGrantPermission' + operationId: post_grant + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqGrant' tags: - - resource - requestBody: - required: true - content: - application/json: - schema: - required: - - job_id - type: object - properties: - engine_name: - type: string - example: 202204261616175720130 + - permission + /permission/query: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: [{"job_id":"202204261616175720130","party_id":"20001","resource_in_use":true,"resource_return_status":ture,"role":"guest"}] - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: no found job - - '/permission/grant/privilege': - post: - summary: grant privilege + description: Success + schema: + $ref: '#/definitions/ResponsePermissionModel' + operationId: get_permission_Query + parameters: + - required: true + description: App ID + in: query + name: app_id + type: string tags: - permission - requestBody: - required: true - content: - application/json: - schema: - required: - - src_role - - src_party_id - type: object - properties: - src_role: - type: string - example: "guest" - src_party_id: - type: string - example: 9999 - privilege_role: - type: string - description: multiple separated by ",". like "guest, host, arbiter" or "all" - example: all - privilege_component: - type: string - description: multiple separated by ",". like "create, run, stop" or "all" - example: all - privilege_command: - type: string - description: multiple separated bu ",". like "reader, dataio, ..." or "all" - example: all - + /permission/resource/delete: + post: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: error - - '/permission/delete/privilege': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_delete_resource_permission + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqResourceGrant' + tags: + - permission + /permission/resource/grant: post: - summary: delete privilege + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_grant_resource_permission + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqResourceGrant' tags: - permission - requestBody: - required: true - content: - application/json: - schema: - required: - - src_role - - src_party_id - type: object - properties: - src_role: - type: string - example: "guest" - src_party_id: - type: string - example: 9999 - privilege_role: - type: string - description: multiple separated by ",". like "guest, host, arbiter" or "all" - example: all - privilege_component: - type: string - description: multiple separated by ",". like "create, run, stop" or "all" - example: all - privilege_command: - type: string - description: multiple separated bu ",". like "reader, dataio, ..." or "all" - example: all - + /permission/resource/query: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: error - - '/permission/query/privilege': - post: - summary: query privilege + description: Success + schema: + $ref: '#/definitions/ResponseResourceModel' + operationId: get_permission_QueryResourcePrivilege + parameters: + - required: true + description: Site ID + in: query + name: party_id + type: string + - required: false + description: Component name + in: query + name: component + type: string + - required: false + description: List of datasets + in: query + name: dataset + type: string tags: - permission - requestBody: - required: true - content: - application/json: - schema: - required: - - src_role - - src_party_id - type: object - properties: - src_role: - type: string - example: "guest" - src_party_id: - type: string - example: 9999 + /permission/role/query: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - example: {"privilege_command":["stop","run","create"],"privilege_component":["reader", "dataio","heterolinr", "heterolr", "localbaseline","columnexpand","heteropearson","featurescale","datastatistics","feldmanverifiablesum"],"privilege_role":["host","guest","arbiter"],"role":"guest","src_party_id":"9999"} - '404': - description: failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 101 - retmsg: - type: string - example: error - - - '/info/fateboard': + description: Success + schema: + $ref: '#/definitions/CommonListModel' + operationId: get_permission_QueryRoles + tags: + - permission + /provider/delete: post: - summary: get fateboard host and port + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseGrantPermission' + operationId: post_provider_Delete + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqProviderDelete' tags: - - information + - provider + /provider/query: + get: responses: '200': - description: fateboard host and port - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - properties: - host: - type: string - example: 127.0.0.1 - port: - type: integer - example: 8080 - '404': - description: fateboard is not configured - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 404 - retmsg: - type: string - example: fateboard is not configured - - '/info/mysql': - post: - summary: test mysql connection + description: Success + schema: + $ref: '#/definitions/CommonDictModel' + operationId: get_provider_Query + parameters: + - required: false + description: Component provider name + in: query + name: name + type: string + - required: false + description: Component running mode + in: query + name: device + type: string + - required: false + description: Component version + in: query + name: version + type: string + - required: false + description: Registered algorithm full name, provider + ':' + version + '@' + running mode, e.g., fate:2.0.0@local + in: query + name: provider_name + type: string tags: - - information + - provider + /provider/register: + post: responses: '200': - description: connect to mysql successfully - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - '404': - description: mysql only available on cluster mode - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 404 - retmsg: - type: string - example: mysql only available on cluster mode - '503': - description: connect to mysql failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 503 - retmsg: - type: string - description: error message - - '/info/eggroll': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_register + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqProviderRegister' + tags: + - provider + /server/delete: post: - summary: test eggroll connection + responses: + '200': + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_delete_server + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqServerDelete' tags: - - information + - server + /server/fateflow: + get: responses: '200': - description: connect to eggroll successfully - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - '404': - description: eggroll only available on cluster mode - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 404 - retmsg: - type: string - example: eggroll only available on cluster mode - '503': - description: connect to eggroll failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 503 - retmsg: - type: string - description: error message - - '/model/deploy': - post: - summary: deploy a model for predict + description: Success + schema: + $ref: '#/definitions/ResponseFateFlow' + operationId: get_server_FateFlowServerInfo tags: - - model - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - model_id - - model_version - properties: - model_id: - type: string - example: 'arbiter-10000#guest-9999#host-10000#model' - model_version: - type: string - example: '202111032155391167400' - components_checkpoint: - type: object - description: specify a checkpoint model to replace the pipeline model - example: - hetero_lr_0: - step_index: 5 - additionalProperties: - type: object - description: use step_index or step_name to specity a checkpoint - properties: - step_index: - type: integer - example: 5 - step_name: - type: string - example: round_5 + - server + /server/query: + get: responses: '200': - description: success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - properties: - model_id: - type: string - example: 'arbiter-9999#guest-10000#host-9999#model' - model_version: - type: string - example: '202111032227378766180' - arbiter: - type: object - properties: - party_id: - type: integer - example: 9999 - guest: - type: object - properties: - party_id: - type: integer - example: 10000 - host: - type: object - properties: - party_id: - type: integer - example: 9999 - detail: - type: object - properties: - arbiter: - type: object - properties: - party_id: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: deploy model of role arbiter 9999 success - guest: - type: object - properties: - party_id: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: deploy model of role guest 10000 success - host: - type: object - properties: - party_id: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: deploy model of role host 9999 success - - '/model/transfer/{model_id}/{model_version}': - post: - summary: download a model by model_id and model_version + description: Success + schema: + $ref: '#/definitions/CommonListModel' + operationId: get_server_QueryServer + parameters: + - required: true + description: Server name + in: query + name: server_name + type: string tags: - - model + - server + /server/query/all: + get: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseFateFlow' + operationId: get_server_QueryAll + tags: + - server + /server/registry: + post: + responses: + '200': + description: Success + schema: + $ref: '#/definitions/ResponseServerRegister' + operationId: post_register_server parameters: - - name: model_id - in: path + - name: payload required: true - description: 'model id (replace # with ~)' + in: body + schema: + $ref: '#/definitions/ReqServerRegistry' + tags: + - server + /server/service/delete: + post: + responses: + '200': + description: Success schema: - type : string - example: 'host~10000~arbiter-10000~guest-9999~host-10000~model' - - name: model_version - in: path + $ref: '#/definitions/CommonModel' + operationId: post_delete_service + parameters: + - name: payload required: true - description: model version + in: body schema: - type : string - example: '202105060929263278441' + $ref: '#/definitions/ReqServiceRegistry' + tags: + - server + /server/service/query: + get: responses: '200': - description: model data - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - description: base64 encoded model data - '404': - description: model not found - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 404 - retmsg: - type: string - example: model not found - - '/model/load': - post: - summary: load a deployed model on Fate-Serving + description: Success + schema: + $ref: '#/definitions/ResponseFateFlow' + operationId: get_server_QueryService + parameters: + - required: true + description: Server name + in: query + name: server_name + type: string + - required: true + description: Service name + in: query + name: service_name + type: string tags: - - model - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - initiator - - role - - job_parameters - properties: - initiator: - type: object - properties: - party_id: - type: string - example: '10000' - role: - type: string - example: guest - role: - type: object - example: - guest: - - '10000' - host: - - '10000' - arbiter: - - '10000' - job_parameters: - type: object - properties: - model_id: - type: string - example: arbiter-10000#guest-10000#host-10000#model - model_version: - type: string - example: '2019081217340125761469' + - server + /server/service/registry: + post: responses: '200': - description: loading success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - jobId: - type: string - example: "2019081217340125761469" - data: - type: object - example: - detail: - guest: - '9999': - retcode: 0 - retmsg: success - host: - '10000': - retcode: 0 - retmsg: success - guest: - '9999': 0 - host: - '10000': 0 - - '/model/bind': - post: - summary: bind a deployed model to Fate-Serving + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_registry_service + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqServiceRegistry' tags: - - model - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - service_id - - initiator - - role - - job_parameters - properties: - service_id: - type: string - example: "123" - initiator: - type: object - properties: - party_id: - type: string - example: "10000" - role: - type: string - example: guest - job_parameters: - type: object - properties: - model_id: - type: string - example: arbiter-10000#guest-10000#host-10000#model - model_version: - type: string - example: "2019081217340125761469" + - server + /site/info/query: + get: responses: '200': - description: binding success - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: service id is 123 - - '/checkpoint/list': - post: - summary: list checkpoints + description: Success + schema: + $ref: '#/definitions/ResponseFateFlow' + operationId: get_site_QuerySiteInfo tags: - - checkpoint - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - model_id - - model_version - - role - - party_id - - component_name - properties: - model_id: - type: string - example: 'arbiter-10000#guest-9999#host-10000#model' - model_version: - type: string - example: '202111032155391167400' - role: - type: string - example: guest - party_id: - type: integer - example: 9999 - component_name: - type: string - example: hetero_lr_0 + - site + /table/bind/path: + post: responses: '200': - description: checkpoints list - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: array - items: - type: object - properties: - create_time: - type: string - example: '2021-11-07T02:34:54.683015' - step_index: - type: integer - example: 0 - step_name: - type: string - example: step_name - models: - type: object - example: - HeteroLogisticRegressionMeta: - buffer_name: LRModelMeta - sha1: 6871508f6e6228341b18031b3623f99a53a87147 - HeteroLogisticRegressionParam: - buffer_name: LRModelParam - sha1: e3cb636fc93675684bff27117943f5bfa87f3029 - additionalProperties: - type: object - properties: - buffer_name: - type: string - example: HeteroLogisticRegressionMeta - sha1: - type: string - example: 6871508f6e6228341b18031b3623f99a53a87147 - - '/checkpoint/get': - post: - summary: get a checkpoint + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_bind_path + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqBindPath' tags: - - checkpoint - requestBody: - required: true - content: - application/json: - schema: - type: object - required: - - model_id - - model_version - - role - - party_id - - component_name - properties: - model_id: - type: string - example: 'arbiter-10000#guest-9999#host-10000#model' - model_version: - type: string - example: '202111032155391167400' - role: - type: string - example: guest - party_id: - type: integer - example: 9999 - component_name: - type: string - example: hetero_lr_0 - step_index: - type: integer - example: 0 + - table + /table/delete: + post: responses: '200': - description: checkpoint data - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - data: - type: object - properties: - create_time: - type: string - example: '2021-11-07T02:34:54.683015' - step_index: - type: integer - example: 0 - step_name: - type: string - example: step_name - models: - type: object - example: - HeteroLogisticRegressionMeta: 'CgJMMhEtQxzr4jYaPxkAAAAAAADwPyIHcm1zcHJvcDD///////////8BOTMzMzMzM8M/QApKBGRpZmZYAQ==' - HeteroLogisticRegressionParam: 'Ig0KAng3EW1qASu+uuO/Ig0KAng0EcNi7a65ReG/Ig0KAng4EbJbl4gvVea/Ig0KAng2EcZwlVZTkOu/Ig0KAngwEVpG8dCbGvG/Ig0KAng5ESJNTx5MLve/Ig0KAngzEZ88H9P8qfO/Ig0KAng1EVfWP8JJv/K/Ig0KAngxEVS0xVXoTem/Ig0KAngyEaApgW32Q/K/KSiiE8AukPs/MgJ4MDICeDEyAngyMgJ4MzICeDQyAng1MgJ4NjICeDcyAng4MgJ4OUj///////////8B' - additionalProperties: - type: string - description: base64 encoded model data - '404': - description: checkpoint not found - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 404 - retmsg: - type: string - example: The checkpoint was not found. - - '/component/validate': - post: - summary: validate component parameters + description: Success + schema: + $ref: '#/definitions/CommonModel' + operationId: post_delete_table + parameters: + - name: payload + required: true + in: body + schema: + $ref: '#/definitions/ReqServiceRegistry' tags: - - component - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - dsl_version: - type: integer - example: 2 - component_name: - type: string - example: dataio_0 - component_module_name: - type: string - example: DataIO - role: - type: object - example: - guest: - - 10000 - host: - - 9999 - arbiter: - - 9999 - component_parameters: - type: object - description: required if `dsl_version` == 2 - example: - common: - dataio_0: - output_format: dense - role: - guest: - '0': - dataio_0: - with_label: true - host: - '0': - dataio_0: - with_label: false - output_format: dense - role_parameters: - type: object - description: required if `dsl_version` == 1 - example: - guest: - dataio_0: - with_label: - - true - label_name: - - y - label_type: - - int - output_format: - - dense - missing_fill: - - true - outlier_replace: - - true - host: - dataio_0: - with_label: - - false - output_format: - - dense - outlier_replace: - - true - algorithm_parameters: - type: object - description: required if `dsl_version` == 1 - example: - hetero_feature_binning_0: - method: quantile - compress_thres: 10000 - head_size: 10000 - error: 0.001 - bin_num: 10 - adjustment_factor: 0.5 - local_only: false - transform_param: - transform_cols: -1 - transform_type: bin_num + - table + /table/query: + get: responses: '200': - description: validation passed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 0 - retmsg: - type: string - example: success - '400': - description: validation failed - content: - application/json: - schema: - type: object - properties: - retcode: - type: integer - example: 400 - retmsg: - type: string - description: error message - example: "Component dataio_0, module DataIO, does not pass component check, error msg is dataio param's dendse not supported, should be one of ['dense', 'sparse']" - + description: Success + schema: + $ref: '#/definitions/ResponseFateFlow' + operationId: get_table_QueryTable + parameters: + - required: true + description: Namespace of the data table + in: query + name: namespace + type: string + - required: true + description: Name of the data table + in: query + name: name + type: string + - required: false + description: Whether to return preview data + in: query + name: display + type: string + tags: + - table +info: + title: FATE Flow restful api + version: 2.0.0 +produces: + - application/json +consumes: + - application/json tags: - - name: data-access - - name: table + - name: client + description: client-Related Operations + - name: data + description: data-Related Operations - name: job - - name: tracking - - name: information + description: job-Related Operations + - name: log + description: log-Related Operations - name: model - - name: checkpoint - - name: component - -servers: - - description: Default Server URL - url: http://localhost:9380/v1 + description: model-Related Operations + - name: output + description: output-Related Operations + - name: permission + description: permission-Related Operations + - name: provider + description: provider-Related Operations + - name: server + description: server-Related Operations + - name: site + description: site-Related Operations + - name: table + description: table-Related Operations +definitions: + ReqAppName: + required: + - app_name + properties: + app_name: + type: string + description: App name for the client + type: object + ResponseClientInfo: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/ClientInfo' + type: object + CommonModel: + properties: + message: + type: string + code: + type: integer + type: object + ClientInfo: + properties: + app_name: + type: string + app_id: + type: string + app_token: + type: string + app_type: + type: string + type: object + ReqAppId: + required: + - app_id + properties: + app_id: + type: string + description: App ID for the client + type: object + ResponseStatus: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/Status' + type: object + Status: + properties: + status: + type: integer + type: object + ReqPartyId: + required: + - party_id + properties: + party_id: + type: string + description: Site ID + type: object + ReqPartnerCreate: + required: + - app_id + - app_token + - party_id + properties: + party_id: + type: string + description: Site ID + app_id: + type: string + description: App ID for the site + app_token: + type: string + description: App token for the site + type: object + ResponsePartnerInfo: + properties: + message: + type: string + code: + type: integer + x-mask: '{data}' + type: object + ReqComponentUpload: + required: + - file + - head + - partitions + properties: + file: + type: string + description: File path on the server + head: + type: boolean + description: Whether the first row of the file is the data's head + partitions: + type: integer + description: Number of data partitions + meta: + type: object + description: Metadata of the data + extend_sid: + type: boolean + description: Whether to automatically fill a column as data row ID + namespace: + type: string + description: Namespace of the data table + name: + type: string + description: Name of the data table + type: object + ResponseUploadData: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/dataInfo' + job_id: + type: string + type: object + dataInfo: + properties: + name: + type: string + namespace: + type: string + type: object + ReqComponentUploadFile: + required: + - head + - partitions + properties: + head: + type: boolean + description: Whether the first row of the file is the data's head + partitions: + type: integer + description: Number of data partitions + meta: + type: object + description: Metadata of the data + extend_sid: + type: boolean + description: Whether to automatically fill a column as data row ID + namespace: + type: string + description: Namespace of the data table + name: + type: string + description: Name of the data table + type: object + ReqComponentDownload: + required: + - name + - namespace + properties: + namespace: + type: string + description: Namespace of the data table + name: + type: string + description: Name of the data table + path: + type: string + description: File path on the server + type: object + ResponseComponentDownload: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/ComponentInfo' + job_id: + type: string + type: object + ComponentInfo: + properties: + name: + type: string + namespace: + type: string + path: + type: string + type: object + ReqTransformerData: + required: + - name + - namespace + properties: + data_warehouse: + type: object + description: 'Data output, content like: {name: xxx, namespace: xxx}' + namespace: + type: string + description: Namespace of the data table + name: + type: string + description: Name of the data table + site_name: + type: string + description: Site name + drop: + type: boolean + description: Whether to destroy data if it already exists + type: object + ReqSubmitJob: + required: + - dag_schema + properties: + dag_schema: + type: object + description: Definition and configuration of jobs, including the configuration of multiple tasks + user_name: + type: string + description: Name of the data table + type: object + ResponseJobSubmit: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/JobInfo' + job_id: + type: string + type: object + JobInfo: + properties: + model_id: + type: string + model_version: + type: string + type: object + CommonListToDictModel: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + type: array + items: + type: object + type: object + ReqJobId: + required: + - job_id + properties: + job_id: + type: string + description: Job ID + type: object + CommonDictModel: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + type: object + type: object + ResponseJobClean: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/queueInfo' + type: object + queueInfo: + properties: + job_id: + type: string + type: object + ReqDataView: + required: + - job_id + - party_id + - role + properties: + job_id: + type: string + description: Job ID + role: + type: string + description: 'Role of the participant: guest/host/arbiter/local' + party_id: + type: string + description: Site ID + type: object + ReqNodesAdd: + required: + - job_id + - notes + - party_id + - role + properties: + job_id: + type: string + description: Job ID + role: + type: string + description: 'Role of the participant: guest/host/arbiter/local' + party_id: + type: string + description: Site ID + notes: + type: string + description: Tags and customizable information for tasks + type: object + CommonDataModel: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + type: integer + type: object + CommonListDictModel: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + type: array + items: + type: object + type: object + ReqModelExport: + required: + - model_id + - model_version + - party_id + - path + - role + properties: + model_id: + type: string + description: Model ID + model_version: + type: string + description: Model version + party_id: + type: string + description: Site ID + role: + type: string + description: 'Role of the participant: guest/host/arbiter/local' + path: + type: string + description: Directory path on the server + type: object + ReqModelImport: + required: + - model_id + - model_version + properties: + model_id: + type: string + description: Model ID + model_version: + type: string + description: Model version + type: object + ReqModelDelete: + allOf: + - $ref: '#/definitions/ReqModelImport' + - properties: + role: + type: string + description: 'Role of the participant: guest/host/arbiter/local' + party_id: + type: string + description: Site ID + task_name: + type: string + description: Task name + output_key: + type: string + description: Primary key for output data or model of the task + type: object + ModelDelete: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/Delete' + type: object + Delete: + properties: + count: + type: boolean + type: object + OutputQuery: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/MetricKeyInfo' + type: object + MetricKeyInfo: + properties: + name: + type: string + step_axis: + type: string + type: + type: string + groups: + type: array + items: + type: object + type: object + ReqMetricDelete: + required: + - job_id + - role + - task_name + properties: + job_id: + type: string + description: Job ID + role: + type: string + description: 'Role of the participant: guest/host/arbiter/local' + task_name: + type: string + description: Task name + type: object + ResponseDeleteMetric: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + type: boolean + type: object + ResponseFateFlow: + properties: + code: + type: integer + message: + type: string + data: + type: object + type: object + ResponseDataTable: + properties: + train_out_data: + type: array + items: + $ref: '#/definitions/dataInfo' + type: object + ResponseTableDisplay: + properties: + train_out_data: + type: array + items: + type: array + items: + type: string + type: object + ReqGrant: + required: + - app_id + - role + properties: + app_id: + type: string + description: App ID + role: + type: string + description: Permission name + type: object + ResponseGrantPermission: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/GrantInfo' + type: object + GrantInfo: + properties: + status: + type: boolean + type: object + ResponsePermissionModel: + properties: + data: + $ref: '#/definitions/PermissionInfo' + type: object + PermissionInfo: + properties: + client: + type: array + items: + type: object + type: object + CommonListModel: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + type: array + items: + type: string + type: object + ReqResourceGrant: + required: + - party_id + properties: + party_id: + type: string + description: Site ID + component: + type: string + description: Component name + dataset: + type: array + items: + type: object + type: object + ResponseResourceModel: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + type: object + type: object + ReqProviderRegister: + required: + - device + - metadata + - name + - version + properties: + name: + type: string + description: Component provider name + device: + type: string + description: Component running mode + version: + type: string + description: Component version + metadata: + type: string + description: Detailed information about component registration + protocol: + type: string + description: 'Protocol: fate/bfia,etc.' + components_description: + type: string + description: Components description + type: object + ReqProviderDelete: + properties: + name: + type: string + description: Component provider name + device: + type: string + description: Component running mode + version: + type: string + description: Component version + provider_name: + type: string + description: Registered algorithm full name, provider + ':' + version + '@' + running mode, e.g., fate:2.0.0@local + type: object + ReqServerRegistry: + required: + - host + - port + - server_name + properties: + server_name: + type: string + description: Server name + host: + type: string + description: Host IP + port: + type: string + description: Service port + protocol: + type: string + description: 'Protocol: fate/bfia,etc.' + type: object + ResponseServerRegister: + allOf: + - $ref: '#/definitions/CommonModel' + - properties: + data: + $ref: '#/definitions/RegisterInfo' + type: object + RegisterInfo: + properties: + host: + type: string + port: + type: string + protocol: + type: string + server_name: + type: string + type: object + ReqServerDelete: + required: + - server_name + properties: + server_name: + type: string + description: Server name + type: object + ReqServiceRegistry: + required: + - name + - namespace + properties: + namespace: + type: string + description: Namespace of the data table + name: + type: string + description: Name of the data table + type: object + ReqBindPath: + required: + - name + - namespace + - path + properties: + namespace: + type: string + description: Namespace of the data table + name: + type: string + description: Name of the data table + path: + type: string + description: File path on the server + type: object +responses: + ParseError: + description: When a mask can't be parsed + MaskError: + description: When any error occurs on mask diff --git a/doc/system_conf.md b/doc/system_conf.md new file mode 100644 index 000000000..d8dc45843 --- /dev/null +++ b/doc/system_conf.md @@ -0,0 +1,215 @@ +# System Configuration +FATE Flow uses YAML to define system configurations, and the configuration file is located at: `conf/service_conf.yaml`. The specific configuration contents and their meanings are as follows: + +| Configuration Item | Description | Values | +|--------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------| +| party_id | Local site ID | For example, "9999", "10000" | +| log_level | Log level | DEBUG:10, INFO:20, DEBUG:30, ERROR: 40 | +| use_registry | Whether to use a registry center; currently, only ZooKeeper mode is supported, and it requires correct ZooKeeper configuration. Note: If using high availability mode, ensure this configuration is set to true. | true/false | +| encrypt | Encryption module | See [Encryption Module](#encryption-module) | +| fateflow | Configuration for the FATE Flow service, including ports, command channel service, and proxy | See [FateFlow Configuration](#fateflow-configuration) | +| database | Configuration information for the database service | See [Database Configuration](#database-configuration) | +| default_engines | System's engine services, including computing, storage, and communication engines | See [Engine Configuration](#engine-configuration) | +| default_provider | Component source information, including provider name, component version, and execution mode | See [Default Registered Algorithm Configuration](#default-registered-algorithm-configuration) | +| federation | Communication service pool | See [Communication Engine Pool](#communication-engine-pool) | +| computing | Computing service pool | See [Computing Engine Pool](#computing-engine-pool) | +| storage | Storage service pool | See [Storage Engine Pool](#storage-configuration) | +| hook_module | Hook configuration, currently supports client authentication, site authentication, and authorization hooks | See [Hook Module Configuration](#hook-module-configuration) | +| authentication | Authentication and authorization switches | See [Authentication Switch](#authentication-switch) | +| model_store | Model storage configuration | See [Model Storage](#model-storage) | +| zookeeper | ZooKeeper service configuration | See [ZooKeeper Configuration](#zookeeper-configuration) | + +## Encryption Module +```yaml +key_0: + module: fate_flow.hub.encrypt.password_encrypt#pwdecrypt + private_path: private_key.pem +``` +This encryption module is primarily used for encrypting passwords (e.g., MySQL passwords): +- "key_0" is the key for the encryption module (you can customize the name), making it easier to reference in other configurations when multiple encryption modes coexist. + - module: The encryption module, formatted as "encryption module" + "#" + "encryption function." + - private_path: The path to the encryption key. If you provide a relative path, its root directory is `fate_flow/conf/`. + +## FateFlow Configuration +```yaml +host: 127.0.0.1 +http_port: 9380 +grpc_port: 9360 +proxy_name: osx +nginx: + host: + http_port: + grpc_port: +``` +- host: Host address. +- http_port: HTTP port number. +- grpc_port: gRPC port number. +- proxy_name: Command channel service name, supporting osx/nginx. Detailed configurations need to be set within [Communication Engine Pool](#communication-engine-pool). +- nginx: Proxy service configuration for load balancing. + +## Database Configuration +```yaml +engine: sqlite +decrypt_key: +mysql: + name: fate_flow + user: fate + passwd: fate + host: 127.0.0.1 + port: 3306 + max_connections: 100 + stale_timeout: 30 +sqlite: + path: +``` +- engine: Database engine name. If set to "mysql" here, update the detailed MySQL configuration. +- decrypt_key: Encryption module, selected from [Encryption Module](#encryption-module). If not configured, it's considered to not use password encryption. If used, you need to set the "passwd" below to ciphertext and configure the key path in [Encryption Module](#encryption-module). +- mysql: MySQL service configuration. If using password encryption functionality, set the "passwd" in this configuration to ciphertext and configure the key path in [Encryption Module](#encryption-module). +- sqlite: SQLite file path, default path is `fate_flow/fate_flow_sqlite.db`. + +## Engine Configuration +```yaml +default_engines: + computing: standalone + federation: standalone + storage: standalone +``` + +- computing: Computing engine, supports "standalone", "eggroll", "spark". +- federation: Communication engine, supports "standalone", "osx", "rabbitmq", "pulsar". +- storage: Storage engine, supports "standalone," "eggroll," "hdfs." + +## Default Registered Algorithm Configuration +- name: Algorithm name. +- version: Algorithm version. If not configured, it uses the configuration in `fateflow.env`. +- device: Algorithm launch mode, local/docker/k8s, etc. + +## Communication Engine Pool +### Pulsar +```yaml +pulsar: + host: 192.168.0.5 + port: 6650 + mng_port: 8080 + cluster: standalone + tenant: fl-tenant + topic_ttl: 30 + route_table: + mode: replication + max_message_size: 1048576 +``` +### Nginx: +```yaml +nginx: + host: 127.0.0.1 + http_port: 9300 + grpc_port: 9310 + protocol: http +``` + +### RabbitMQ +```yaml +nginx: + host: 127.0.0.1 + http_port: 9300 + grpc_port: 9310 + protocol: http +``` + +### OSx +```yaml + host: 127.0.0.1 + port: 9370 + mode: stream +``` + +## Computing Engine Pool +### Standalone +```yaml + cores: 32 +``` +- cores: Total resources. + +### Eggroll +```yaml +eggroll: + cores: 32 + nodes: 1 + host: 127.0.0.1 + port: 4670 +``` +- cores: Total cluster resources. +- nodes: Number of node managers in the cluster. +- host: eggroll cluster manager host ip +- port: eggroll cluster manager port + +### Spark +```yaml +eggroll: + home: + cores: 32 +``` +- home: Spark home directory. If not filled, "pyspark" will be used as the computing engine. +- cores: Total resources. + +## Storage Engine Pool +```yaml + hdfs: + name_node: hdfs://fate-cluster +``` + +## Hook Module Configuration +```yaml +hook_module: + client_authentication: fate_flow.hook.flow.client_authentication + site_authentication: fate_flow.hook.flow.site_authentication + permission: fate_flow.hook.flow.permission +``` +- client_authentication: Client authentication hook. +- site_authentication: Site authentication hook. +- permission: Permission authentication hook. + +## Authentication Switch +```yaml +authentication: + client: false + site: false + permission: false +``` + +## Model Storage +```yaml +model_store: + engine: file + decrypt_key: + file: + path: + mysql: + name: fate_flow + user: fate + passwd: fate + host: 127.0.0.1 + port: 3306 + max_connections: 100 + stale_timeout: 30 + tencent_cos: + Region: + SecretId: + SecretKey: + Bucket: +``` +- engine: Model storage engine, supports "file," "mysql", and "tencent_cos". +- decrypt_key: Encryption module, needs to be selected from [Encryption Module](#encryption-module). If not configured, it is assumed to not use password encryption. If used, you need to set the "passwd" below accordingly to ciphertext and configure the key path in [Encryption Module](#encryption-module). +- file: Model storage directory, default location is `fate_flow/model`. +- mysql: MySQL service configuration; if using password encryption functionality, you need to set the "passwd" in this configuration to ciphertext and configure the key path in [Encryption Module](#encryption-module). +- tencent_cos: Tencent Cloud key configuration. + +## ZooKeeper Configuration +```yaml +zookeeper: + hosts: + - 127.0.0.1:2181 + use_acl: true + user: fate + password: fate +``` \ No newline at end of file diff --git a/doc/system_conf.zh.md b/doc/system_conf.zh.md new file mode 100644 index 000000000..5c8f910f5 --- /dev/null +++ b/doc/system_conf.zh.md @@ -0,0 +1,220 @@ +# 系统配置描述文档 +FATE Flow使用yaml定义系统配置，配置路径位于: conf/service_conf.yaml, 具体配置内容及其含义如下： + +| 配置项 | 说明 | 值 | +|------------------|----------------------------------------------------------------------------|----------------------------------------| +| party_id | 本方站点id | 如: "9999", "10000 | +| use_registry | 是否使用注册中心，当前仅支持zookeeper模式，需要保证zookeeper的配置正确；
注：若使用高可用模式，需保证该配置设置为true | true/false | +| log_level | 日志级别 | DEBUG:10, INFO:20, DEBUG:30, ERROR: 40 | +| encrypt | 加密模块 | 见[加密模块](#加密模块) | +| fateflow | FATE Flow服务的配置，主要包括端口、命令通道服务、代理等 | 见[FateFlow配置](#fateflow配置) | +| database | 数据库服务的配置信息 | 见[数据库配置](#数据库配置) | +| default_engines | 系统的引擎服务，主要包括计算、存储和通信引擎 | 见[引擎配置](#引擎配置) | +| default_provider | 组件的来源信息，主要包括提供方名称、组件版本和运行模式 | 见[默认注册算法配置](#默认注册算法配置) | +| federation | 通信服务池 | 见[通信引擎池](#通信引擎池) | +| computing | 计算服务池 | 见[计算引擎池](#计算引擎池) | +| storage | 存储服务池 | 见[存储引擎池](#存储配置) | +| hook_module | 钩子配置，当前支持客户端认证、站点端认证以及鉴权钩子 | 见[钩子模块配置](#钩子模块配置) | +| authentication | 认证&&鉴权开关 | 见[认证开关](#认证开关) | +| model_store | 模型存储配置 | 见[模型存储](#模型存储) | +| zookeeper | zookeeper服务的配置 | 见[zookeeper配置](#zookeeper配置) | + +## 加密模块 +```yaml +key_0: + module: fate_flow.hub.encrypt.password_encrypt#pwdecrypt + private_path: private_key.pem +``` +该加密模块主要用于密码(如mysql密码)等内容加密： +- 其中"key_0"为加密模块的key(可以自定义名字)，便于其它配置中直接引用，多套加密模式共存。 + - module: 加密模块，拼接规则为：加密模块 + "#" + 加密函数。 + - private_path：密钥路径。如填相对路径，其根目录位于fate_flow/conf/ + +## FateFlow配置 +```yaml +host: 127.0.0.1 +http_port: 9380 +grpc_port: 9360 +proxy_name: osx +nginx: + host: + http_port: + grpc_port: +``` +- host: 主机地址; +- http_port：http端口号; +- grpc_port: grpc端口号; +- proxy_name: 命令通道服务名，支持osx/nginx。详细配置需要在[通信引擎池](#通信引擎池) 里面配置; +- nginx: 代理服务配置，用于负载均衡。 + +## 数据库配置 +```yaml +engine: sqlite +decrypt_key: +mysql: + name: fate_flow + user: fate + passwd: fate + host: 127.0.0.1 + port: 3306 + max_connections: 100 + stale_timeout: 30 +sqlite: + path: +``` +- engine: 数据库引擎名字，如这里填"mysql"，则需要更新mysql的配置详细配置。 +- decrypt_key: 加密模块,需要从[加密模块](#加密模块)中选择。 若不配置，视为不使用密码加密；若使用，则需要将下面的passwd相应设置为密文。 +- mysql: mysql服务配置；若使用密码加密功能，需要将此配置中的"passwd"设置为密文，并在[加密模块](#加密模块)中配置密钥路径 +- sqlite: sqlite文件路径，默认路径为fate_flow/fate_flow_sqlite.db + +## 引擎配置 +```yaml +default_engines: + computing: standalone + federation: standalone + storage: standalone +``` + +- computing: 计算引擎，支持"standalone"、"eggroll"、"spark" +- federation: 通信引擎，支持"standalone"、"osx"、"rabbitmq"、"pulsar" +- storage: 存储引擎，支持"standalone"、"eggroll"、"hdfs" + +## 默认注册算法配置 +- name: 算法名称 +- version: 算法版本，若不配置，则使用fateflow.env中的配置 +- device: 算法启动方式, local/docker/k8s等 + +## 通信引擎池 +### pulsar +```yaml +pulsar: + host: 192.168.0.5 + port: 6650 + mng_port: 8080 + cluster: standalone + tenant: fl-tenant + topic_ttl: 30 + # default conf/pulsar_route_table.yaml + route_table: + # mode: replication / client, default: replication + mode: replication + max_message_size: 1048576 +``` +### nginx: +```yaml +nginx: + host: 127.0.0.1 + http_port: 9300 + grpc_port: 9310 + # http or grpc + protocol: http +``` + +### rabbitmq +```yaml +nginx: + host: 127.0.0.1 + http_port: 9300 + grpc_port: 9310 + # http or grpc + protocol: http +``` + +### osx +```yaml + host: 127.0.0.1 + port: 9370 + mode: stream +``` + +## 计算引擎池 +### standalone +```yaml + cores: 32 +``` +- cores: 资源总数 + +### eggroll +```yaml +eggroll: + cores: 32 + nodes: 1 + host: 127.0.0.1 + port: 4670 +``` +- cores: 集群资源总数 +- nodes: 集群node-manager数量 +- host: eggroll cluster manager host ip +- port: eggroll cluster manager port + +### spark +```yaml +eggroll: + home: + cores: 32 +``` +- home: spark home目录，如果不填，将使用"pyspark"作为计算引擎。 +- cores: 资源总数 + +## 存储引擎池 +```yaml + hdfs: + name_node: hdfs://fate-cluster +``` + +## 钩子模块配置 +```yaml +hook_module: + client_authentication: fate_flow.hook.flow.client_authentication + site_authentication: fate_flow.hook.flow.site_authentication + permission: fate_flow.hook.flow.permission +``` +- client_authentication: 客户端认证钩子 +- site_authentication: 站点认证钩子 +- permission: 权限认证钩子 + +## 认证开关 +```yaml +authentication: + client: false + site: false + permission: false +``` + +## 模型存储 +```yaml +model_store: + engine: file + decrypt_key: + file: + path: + mysql: + name: fate_flow + user: fate + passwd: fate + host: 127.0.0.1 + port: 3306 + max_connections: 100 + stale_timeout: 30 + tencent_cos: + Region: + SecretId: + SecretKey: + Bucket: +``` +- engine: 模型存储引擎，支持"file"、"mysql"和"tencent_cos"。 +- decrypt_key: 加密模块,需要从[加密模块](#加密模块)中选择。 若不配置，视为不使用密码加密；若使用，则需要将下面的passwd相应设置为密文。 +- file: 模型存储目录，默认位于： fate_flow/model +- mysql: mysql服务配置；若使用密码加密功能，需要将此配置中的"passwd"设置为密文，并在[加密模块](#加密模块)中配置密钥路径 +- tencent_cos: 腾讯云密钥配置 + + +## zookeeper配置 +```yaml +zookeeper: + hosts: + - 127.0.0.1:2181 + use_acl: true + user: fate + password: fate +``` \ No newline at end of file diff --git a/doc/system_operational.md b/doc/system_operational.md deleted file mode 100644 index 97467029d..000000000 --- a/doc/system_operational.md +++ /dev/null @@ -1,75 +0,0 @@ -# System Operation - -## 1. Description - -## 2. Log cleaning - -## 2.1 Job logs (N=14 days) - -- Machine: the machine where fate flow is located -- Directory: ${FATE_PROJECT_BASE}/fateflow/logs/ -- Rule: directory starts with $jobid, clean up the data before $jobid is **N days** -- Reference command. - -```bash -rm -rf ${FATE_PROJECT_BASE}/fateflow/logs/20200417* -``` - -### 2.2 EggRoll Session logs (N=14 days) - -- Machine: eggroll node -- Directory: ${FATE_PROJECT_BASE}/eggroll/logs/ -- Rule: directory starts with $jobid, clean up data before $jobid is **N days** -- Reference command. - -```bash -rm -rf ${FATE_PROJECT_BASE}/eggroll/logs/20200417* -``` - -### 2.3 fateflow system logs (N=14 days) - -- Machine: fate flow machine -- Directory: ${FATE_PROJECT_BASE}/logs/fate_flow/ -- Rule: Log file ends with yyyy-dd-mm, clean up data before **N days** -- Archive: log file ends with yyyy-dd-mm, archive to keep 180 days of logs -- Reference command. - -```bash -rm -rf ${FATE_PROJECT_BASE}/logs/fate_flow/fate_flow_stat.log.2020-12-15 -``` - -### 2.4 EggRoll system logs (N=14 days) - -- Machine: eggroll deployment machine -- Directory: ${FATE_PROJECT_BASE}/eggroll/logs/eggroll -- Rule: directory is yyyy/mm/dd, clean up data before **N days** -- Archive: directory is yyyy/mm/dd, archive the logs retained for 180 days -- Reference command. - -```bash -rm -rf ${FATE_PROJECT_BASE}/eggroll/logs/2020/12/15/ -``` - -## 3. Data cleanup - -### 3.1 Calculate temporary data (N=2 days) - -- Machine: eggroll node -- Directory: ${FATE_PROJECT_BASE}/eggroll/data/IN_MEMORY -- Rule: namespace starts with $jobid, clean up data before $jobid is **N days** -- Reference command. - -```bash -rm -rf ${FATE_PROJECT_BASE}/eggroll/data/IN_MEMORY/20200417* -``` - -### 3.2 Component output data (N=14 days) - -- Machine: eggroll node -- Directory: ${FATE_PROJECT_BASE}/eggroll/data/LMDB -- Rule: namespace starts with output_data_$jobid, clean up $jobid for data before **N days** -- Reference command. - -```bash -rm -rf ${FATE_PROJECT_BASE}/eggroll/data/LMDB/output_data_20200417* -``` diff --git a/doc/system_operational.zh.md b/doc/system_operational.zh.md deleted file mode 100644 index 7e2ed66d0..000000000 --- a/doc/system_operational.zh.md +++ /dev/null @@ -1,75 +0,0 @@ -# 系统运维 - -## 1. 说明 - -## 2. 日志清理 - -### 2.1 作业日志(N=14天) - -- 所在机器：fate flow所在机器 -- 目录：${FATE_PROJECT_BASE}/fateflow/logs/ -- 规则：目录以$jobid开头，清理$jobid为**N天**前的数据 -- 参考命令： - -```bash -rm -rf ${FATE_PROJECT_BASE}/fateflow/logs/20200417* -``` - -### 2.2 EggRoll Session日志(N=14天) - -- 所在机器：eggroll node节点 -- 目录：${FATE_PROJECT_BASE}/eggroll/logs/ -- 规则：目录以$jobid开头，清理$jobid为**N天**前的数据 -- 参考命令： - -```bash -rm -rf ${FATE_PROJECT_BASE}/eggroll/logs/20200417* -``` - -### 2.3 fateflow 系统日志(N=14天) - -- 所在机器：fate flow所在机器 -- 目录：${FATE_PROJECT_BASE}/logs/fate_flow/ -- 规则：日志文件以yyyy-dd-mm结尾，清理**N天**前的数据 -- 归档：日志文件以yyyy-dd-mm结尾，归档保留180天的日志 -- 参考命令： - -```bash -rm -rf ${FATE_PROJECT_BASE}/logs/fate_flow/fate_flow_stat.log.2020-12-15 -``` - -### 2.4 EggRoll 系统日志(N=14天) - -- 所在机器：eggroll部署机器 -- 目录：${FATE_PROJECT_BASE}/eggroll/logs/eggroll -- 规则：目录为yyyy/mm/dd，清理**N天**前的数据 -- 归档：目录为yyyy/mm/dd，归档保留180天的日志 -- 参考命令： - -```bash -rm -rf ${FATE_PROJECT_BASE}/eggroll/logs/2020/12/15/ -``` - -## 3. 数据清理 - -### 3.1 计算临时数据(N=2天) - -- 所在机器：eggroll node节点 -- 目录：${FATE_PROJECT_BASE}/eggroll/data/IN_MEMORY -- 规则：namespace以$jobid开头，清理$jobid为**N天**前的数据 -- 参考命令： - -```bash -rm -rf ${FATE_PROJECT_BASE}/eggroll/data/IN_MEMORY/20200417* -``` - -### 3.2 组件输出数据(N=14天) - -- 所在机器：eggroll node节点 -- 目录：${FATE_PROJECT_BASE}/eggroll/data/LMDB -- 规则：namespace以output_data_$jobid开头，清理$jobid为**N天**前的数据 -- 参考命令： - -```bash -rm -rf ${FATE_PROJECT_BASE}/eggroll/data/LMDB/output_data_20200417* -``` diff --git a/doc/template/cli_desc.md b/doc/template/cli_desc.md deleted file mode 100644 index a520a2f00..000000000 --- a/doc/template/cli_desc.md +++ /dev/null @@ -1,35 +0,0 @@ -**请求CLI** - -```bash -flow -``` - -**参数** - -| 参数名 | 必选 | 类型 | 说明 | -| :--------------------- | :--- | :----- | ------------------------------| -| -j, --job-id | 是 | string | 作业id | -| -r, --role | 是 | string | 参与角色 | -| -p, --partyid | 是 | string | 参与方id | -| -cpn, --component-name | 是 | string | 组件名，与job dsl中的保持一致 | -| -o, --output-path | 是 | string | 输出数据的存放路径 | - -**返回参数** - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | -| data | dict | 返回数据 | -| jobId | string | 作业id | - -**样例** - -```bash -flow -``` - -输出: - -```json -``` diff --git a/doc/third_party_service_registry.md b/doc/third_party_service_registry.md deleted file mode 100644 index 43026b8fd..000000000 --- a/doc/third_party_service_registry.md +++ /dev/null @@ -1,203 +0,0 @@ -## Third party service registration center - -## 1. Description -- fateflow supports third-party services for registration for callback scenarios -- All interfaces need to register the service address first, then register the interface - -## 2. Registration -## 2.1 Server registration -- uri: ```/v1/server//register``` -- Method: POST -- Request Parameters - -| parameter name | required | type | description | -|:--------|:----|:-------|--------| -| host | yes | string | service ip address | -| port | yes | int | service port | - -- Return parameters - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | return message | - - -## 2.2 Service Registration -- uri: ```/v1/service/registry``` -- Method: POST -- Request Parameters - -| parameter name | required | type | description | -|:----|:-------|:------------|--------| -| server_name | yes | string | The name of the registered server | -| service_name | yes | string | service name | -| uri | yes | string | service uri | -| method | no | string | Request method, default "POST" | -| protocol | no | string | default "http" | - -- Return parameters - -| parameter name | type | description | -| :------ | :----- | -------- | -| retcode | int | return code | -| retmsg | string | Return information | - - -## 3 Interface parameter details -### 3.1 ApiReader -The ApiReader component requires third-party services to register three interfaces: upload, query, download, which are used to request feature data for offline ids. - -#### 3.1.1 upload -- Description: upload interface passes the id to the third-party service -- Interface registration: refer to [service registration](#22-service-registration), where the service_name parameter is "upload". -- Request parameters - - headers: {"Content-Type": "application/octet-stream"} - - params: - -| parameter_name | required | type | description | -|:----|:---------|:-----------------|--------| -| requestBody | yes | string | json string containing feature filtering parameters | - - body: data stream - - - Example request (python). - - ```python - - with open(id_path, "w") as f: - data = MultipartEncoder( - fields={'file': (id_path, f, 'application/octet-stream')} - ) - upload_registry_info = service_info.get("upload") - response = getattr(requests, upload_registry_info.f_method.lower(), None)( - url=upload_registry_info.f_url, - params={"requestBody": json.dumps({"stat_month": "202201", "version": "v1"})}, - data=data, - headers={'Content-Type': "application/octet-stream"} - ) - ``` -- The interface returns: - -| parameter name | type | description | -|:-------| :----- | -------- | -| code | int | return code | -| message | string | Returns the message | -| data | object | Returns the jobId parameter for asynchronous status queries | - -#### 3.1.2 query -- Description: query interface is used to query the progress of a task. -- Interface registration: refer to [Service Registration](#22-service-registration), where the service_name parameter is "query". -- Request parameters - - body - -| parameter_name | mandatory | type | description | -|:----|:-------|:--------------|--------| -| jobId | yes | string | The jobId returned by upload | - -- interface returns: - -| parameter name | type | description | -|:-------|:-----| -------- | -| code | int | Return code | -| message | string | Return message | -| status | string | Task status | - - -#### 3.1.3 download -- Description: query interface for querying the progress of the task -- Interface registration: refer to [Service Registration](#22-service-registration), where the service_name parameter is "download". -- Request parameters - - params - -| parameter_name | mandatory | type | description | -|:----|:-------|:--------------|--------| -| requestBody | is | string | json string containing "jobId" | - -- Interface Return: Feature data stream - - -### 3.2 Authentication -#### 3.2.1 Client authentication (client_authentication) -- Description: Client authentication is used to authenticate client requests -- Interface Registration: Refer to [Service Registration](#22-service-registration), where the service_name parameter is "client_authentication". -- Request parameters. - - body - -| parameter_name | required | type | description -|:----------|:----|:-------|------| -| full_path | yes | string | request path | -| headers | yes | string | request headers | -| form | no | object | request body | - -- Interface Return: - -| parameter name | type | description | -|:-----|:-----| -------- | -| code | int | return code | -| msg | string | return message | - - -#### 3.2.2 Site Authentication -##### 3.2.2.1 signature -- Description: Before requesting another site, fate flow will call the signature interface to get the signature and put it in the request header -- Interface registration: Refer to [Service Registration](#22-service-registration), where the service_name parameter is "signature". -- Request parameters. - - body - -| parameter_name | mandatory | type | description | -|:---------|:----|:-------|------| -| party_id | yes | string | site id | -| body | yes | object | request body | - - -- Interface Return: - -| parameter name | type | description | -|:-----|:-----|-----| -| code | int | return code | -| site_signature | string | signature | - -##### 3.2.2.2 site_authentication -- Description: Used to authenticate requests from other fate sites. -- Interface registration: refer to [Service Registration](#22-service-registration), where the service_name parameter is "site_authentication". -- Request parameters. - - body - -| parameter_name | required | type | description -|:---------------|:----|:-------|---------| -| src_party_id | yes | string | Requesting party site id | -| site_signature | yes | string | signature | -| body | yes | object | request body | - -- Interface Return: - -| parameter name | type | description | -|:-----|:-----| -------- | -| code | int | return code | -| msg | string | return message | - -### 3.3 permission -- Description: Authentication of requests from other sites -- Interface registration: refer to [service registration](#22-service-registration), where the service_name parameter is "permission". -- Request parameters - - body - -| parameter_name | mandatory | type | description -|:----------|:----|:-------|------| -| src_role | yes | string | Requesting party role | -| src_party_id | yes | string | Requesting party partyid | -| initiator | no | object | initiator information | -| roles | no | object | All participant information | -| component_list | yes | object | Component list | -| dataset_list | yes | object | dataset_list | yes | object -| run_time_conf | no | object | job conf | -| dsl | no | object | job dsl | -| component_parameters | no | object | component_parameters | - - -- interface returns: - -| parameter_name | type | description | -|:-----|:-----| -------- | -| code | int | return_code | -| msg | string | return message | \ No newline at end of file diff --git a/doc/third_party_service_registry.zh.md b/doc/third_party_service_registry.zh.md deleted file mode 100644 index 73644f4b7..000000000 --- a/doc/third_party_service_registry.zh.md +++ /dev/null @@ -1,203 +0,0 @@ -# 第三方服务注册中心 - -## 1. 说明 -- fateflow支持第三方服务进行注册，用于回调场景 -- 所有的接口需要先注册服务地址，再注册接口 - -## 2. 注册 -### 2.1 服务器注册 -- uri: ```/v1/server//register``` -- 方式：POST -- 请求参数 - -| 参数名 | 必选 | 类型 | 说明 | -|:--------|:----|:-------|--------| -| host | 是 | string | 服务ip地址 | -| port | 是 | int | 服务端口 | - -- 返回参数 - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | - - -## 2.2 服务注册 -- uri: ```/v1/service/registry``` -- 方式：POST -- 请求参数 - -| 参数名 | 必选 | 类型 | 说明 | -|:----|:-------|:------------|--------| -| server_name | 是 | string | 注册的服务器名字 | -| service_name | 是 | string | 服务名 | -| uri | 是 | string | 服务uri | -| method | 否 | string | 请求方式，默认"POST" | -| protocol | 否 | string | 默认"http" | - -- 返回参数 - -| 参数名 | 类型 | 说明 | -| :------ | :----- | -------- | -| retcode | int | 返回码 | -| retmsg | string | 返回信息 | - - -## 3 接口参数详情 -### 3.1 ApiReader -ApiReader组件需要第三方服务注册三个接口：upload、query、download，用于请求离线id的特征数据。 - -#### 3.1.1 upload -- 说明：upload接口将id传给第三方服务 -- 接口注册：参考[服务注册](#2-服务注册)，其中service_name参数为"upload" -- 请求参数： - - headers: {"Content-Type": "application/octet-stream"} - - params: - -| 参数名 | 必选 | 类型 | 说明 | -|:----|:-------|:-----------------|--------| -| requestBody | 是 | string | json字符串，包含特征筛选参数 | - - body：数据流 - - - 请求示例(python)： - - ```python - - with open(id_path, "w") as f: - data = MultipartEncoder( - fields={'file': (id_path, f, 'application/octet-stream')} - ) - upload_registry_info = service_info.get("upload") - response = getattr(requests, upload_registry_info.f_method.lower(), None)( - url=upload_registry_info.f_url, - params={"requestBody": json.dumps({"stat_month": "202201", "version": "v1"})}, - data=data, - headers={'Content-Type': "application/octet-stream"} - ) - ``` -- 接口返回: - -| 参数名 | 类型 | 说明 | -|:-------| :----- | -------- | -| code | int | 返回码 | -| message | string | 返回信息 | -| data | object | 返回参数jobId，用于异步状态查询 | - -#### 3.1.2 query -- 说明：query接口用于查询任务进度 -- 接口注册：参考[服务注册](#2-服务注册)，其中service_name参数为"query" -- 请求参数： - - body： - -| 参数名 | 必选 | 类型 | 说明 | -|:----|:-------|:--------------|--------| -| jobId | 是 | string | upload返回的jobId | - -- 接口返回: - -| 参数名 | 类型 | 说明 | -|:-------|:-----| -------- | -| code | int | 返回码 | -| message | string | 返回信息 | -| status | string | 任务状态 | - - -#### 3.1.3 download -- 说明：query接口用于查询任务进度 -- 接口注册：参考[服务注册](#2-服务注册)，其中service_name参数为"download" -- 请求参数： - - params： - -| 参数名 | 必选 | 类型 | 说明 | -|:----|:-------|:--------------|--------| -| requestBody | 是 | string | json字符串，包含"jobId" | - -- 接口返回: 特征数据流 - - -### 3.2 认证 -#### 3.2.1 客户端认证(client_authentication) -- 说明：客户端认证用于认证客户端的请求 -- 接口注册：参考[服务注册](#2-服务注册)，其中service_name参数为"client_authentication" -- 请求参数： - - body： - -| 参数名 | 必选 | 类型 | 说明 | -|:----------|:----|:-------|------| -| full_path | 是 | string | 请求路径 | -| headers | 是 | string | 请求头 | -| form | 否 | object | 请求体 | - -- 接口返回: - -| 参数名 | 类型 | 说明 | -|:-----|:-----| -------- | -| code | int | 返回码 | -| msg | string | 返回信息 | - - -#### 3.2.2 站点认证 -##### 3.2.2.1 signature -- 说明：请求其他站点前，fate flow会调用签名接口获取签名并放到请求头中 -- 接口注册：参考[服务注册](#2-服务注册)，其中service_name参数为"signature" -- 请求参数： - - body： - -| 参数名 | 必选 | 类型 | 说明 | -|:---------|:----|:-------|------| -| party_id | 是 | string | 站点id | -| body | 是 | object | 请求体 | - - -- 接口返回: - -| 参数名 | 类型 | 说明 | -|:-----|:-----|-----| -| code | int | 返回码 | -| site_signature | string | 签名 | - -##### 3.2.2.2 site_authentication -- 说明：用于认证其他fate站点的请求 -- 接口注册：参考[服务注册](#2-服务注册)，其中service_name参数为"site_authentication" -- 请求参数： - - body： - -| 参数名 | 必选 | 类型 | 说明 | -|:---------------|:----|:-------|---------| -| src_party_id | 是 | string | 请求方站点id | -| site_signature | 是 | string | 签名 | -| body | 是 | object | 请求体 | - -- 接口返回: - -| 参数名 | 类型 | 说明 | -|:-----|:-----| -------- | -| code | int | 返回码 | -| msg | string | 返回信息 | - -### 3.3 鉴权(permission) -- 说明：对其他站点的请求进行鉴权 -- 接口注册：参考[服务注册](#2-服务注册)，其中service_name参数为"permission" -- 请求参数： - - body： - -| 参数名 | 必选 | 类型 | 说明 | -|:----------|:----|:-------|------| -| src_role | 是 | string | 请求方角色 | -| src_party_id | 是 | string | 请求方partyid | -| initiator | 否 | object | 发起方信息 | -| roles | 否 | object | 全部参与方信息 | -| component_list | 是 | object | 组件列表 | -| dataset_list | 是 | object | 数据集列表 | -| run_time_conf | 否 | object | job conf | -| dsl | 否 | object | job dsl | -| component_parameters | 否 | object | 组件参数 | - - -- 接口返回: - -| 参数名 | 类型 | 说明 | -|:-----|:-----| -------- | -| code | int | 返回码 | -| msg | string | 返回信息 | \ No newline at end of file diff --git a/examples/advanced/job_reference_data_dsl.json b/examples/advanced/job_reference_data_dsl.json deleted file mode 100644 index 07653e6ef..000000000 --- a/examples/advanced/job_reference_data_dsl.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "table" - ] - } - }, - "hetero_feature_binning_0": { - "module": "HeteroFeatureBinning", - "input": { - "data": { - "data": [ - "reader_0.table" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "hetero_feature_binning" - ] - } - } - } -} diff --git a/examples/advanced/job_reference_data_hetero_lr_job_conf.json b/examples/advanced/job_reference_data_hetero_lr_job_conf.json deleted file mode 100644 index 83df2738e..000000000 --- a/examples/advanced/job_reference_data_hetero_lr_job_conf.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "dsl_version": "2", - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "guest": [ - 9999 - ], - "host": [ - 10000 - ], - "arbiter": [ - 10000 - ] - }, - "job_parameters": { - "common": { - "task_parallelism": 1, - "task_cores": 2, - "auto_retries": 1, - "computing_partitions": 8, - "component_provider": "fate_federated_algorithm" - } - }, - "component_parameters": { - "common": { - "intersect_0": { - "intersect_method": "raw", - "sync_intersect_ids": true, - "only_output_key": false - }, - "hetero_lr_0": { - "penalty": "L2", - "optimizer": "rmsprop", - "alpha": 0.01, - "max_iter": 3, - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - } - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "job_id": "202107291950368353361", - "component_name": "data_transform_0", - "data": "train" - } - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "job_id": "202107291950368353361", - "component_name": "data_transform_0", - "data": "train" - } - } - } - } - } - } -} diff --git a/examples/advanced/test_multi_pipeline_conf.json b/examples/advanced/test_multi_pipeline_conf.json deleted file mode 100644 index b0626a087..000000000 --- a/examples/advanced/test_multi_pipeline_conf.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "dsl_version": "2", - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "guest": [ - 9999 - ], - "host": [ - 10000 - ], - "arbiter": [ - 10000 - ] - }, - "job_parameters": { - "common": { - "task_parallelism": 2, - "computing_partitions": 8, - "task_cores": 2 - } - }, - "component_parameters": { - "common": { - "federated_sample_0": { - "mode": "stratified", - "method": "upsample", - "fractions": [ - [ - 0, - 1.5 - ], - [ - 1, - 2.0 - ] - ] - }, - "hetero_lr_0": { - "penalty": "L2", - "optimizer": "rmsprop", - "alpha": 0.01, - "max_iter": 3, - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - } - }, - "hetero_lr_1": { - "penalty": "L2", - "optimizer": "rmsprop", - "alpha": 0.01, - "max_iter": 3, - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - }, - "cv_param": { - "n_splits": 2, - "shuffle": true, - "random_seed": 103, - "need_cv": true - } - }, - "hetero_secureboost_0": { - "cv_param": { - "n_splits": 2, - "need_cv": true, - "shuffle": false - }, - "num_trees": 3 - }, - "hetero_secureboost_1": { - "num_trees": 2 - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": true, - "label_name": "y", - "label_type": "int", - "output_format": "dense", - "missing_fill": true, - "outlier_replace": true - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": false, - "output_format": "dense", - "outlier_replace": true - } - } - } - } - } -} diff --git a/examples/advanced/test_multi_pipeline_conf_advanced.json b/examples/advanced/test_multi_pipeline_conf_advanced.json deleted file mode 100644 index 98e182f2f..000000000 --- a/examples/advanced/test_multi_pipeline_conf_advanced.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "dsl_version": "2", - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "guest": [ - 9999 - ], - "host": [ - 10000 - ], - "arbiter": [ - 10000 - ] - }, - "job_parameters": { - "common": { - "task_parallelism": 2, - "computing_partitions": 8, - "task_cores": 4, - "spark_run": { - "num-executors": 1, - "executor-cores": 2 - } - } - }, - "component_parameters": { - "common": { - "federated_sample_0": { - "mode": "stratified", - "method": "upsample", - "fractions": [ - [ - 0, - 1.5 - ], - [ - 1, - 2.0 - ] - ] - }, - "hetero_lr_0": { - "penalty": "L2", - "optimizer": "rmsprop", - "alpha": 0.01, - "max_iter": 3, - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - } - }, - "hetero_lr_1": { - "penalty": "L2", - "optimizer": "rmsprop", - "alpha": 0.01, - "max_iter": 3, - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - }, - "cv_param": { - "n_splits": 2, - "shuffle": true, - "random_seed": 103, - "need_cv": true - } - }, - "hetero_secureboost_0": { - "cv_param": { - "n_splits": 2, - "need_cv": true, - "shuffle": false - }, - "num_trees": 3 - }, - "hetero_secureboost_1": { - "num_trees": 2 - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": true, - "label_name": "y", - "label_type": "int", - "output_format": "dense", - "missing_fill": true, - "outlier_replace": true - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": false, - "output_format": "dense", - "outlier_replace": true - } - } - } - } - } -} diff --git a/examples/advanced/test_multi_pipeline_dsl.json b/examples/advanced/test_multi_pipeline_dsl.json deleted file mode 100644 index cd3eb4129..000000000 --- a/examples/advanced/test_multi_pipeline_dsl.json +++ /dev/null @@ -1,224 +0,0 @@ -{ - "components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "table" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.table" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "data_transform" - ] - } - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ] - } - }, - "federated_sample_0": { - "module": "FederatedSample", - "input": { - "data": { - "data": [ - "intersection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ] - } - }, - "feature_scale_0": { - "module": "FeatureScale", - "input": { - "data": { - "data": [ - "federated_sample_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "feature_scale" - ] - } - }, - "hetero_feature_binning_0": { - "module": "HeteroFeatureBinning", - "input": { - "data": { - "data": [ - "feature_scale_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "hetero_feature_binning" - ] - } - }, - "hetero_feature_selection_0": { - "module": "HeteroFeatureSelection", - "input": { - "data": { - "data": [ - "hetero_feature_binning_0.train" - ] - }, - "isometric_model": [ - "hetero_feature_binning_0.hetero_feature_binning" - ] - }, - "output": { - "data": [ - "train" - ], - "model": [ - "selected" - ] - } - }, - "one_hot_0": { - "module": "OneHotEncoder", - "input": { - "data": { - "data": [ - "hetero_feature_selection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "onehot_encoder" - ] - } - }, - "hetero_lr_0": { - "module": "HeteroLR", - "input": { - "data": { - "train_data": [ - "one_hot_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "hetero_lr" - ] - }, - "need_deploy": false - }, - "hetero_lr_1": { - "module": "HeteroLR", - "input": { - "data": { - "train_data": [ - "one_hot_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ] - }, - "need_deploy": false - }, - "hetero_secureboost_0": { - "module": "HeteroSecureBoost", - "input": { - "data": { - "train_data": [ - "one_hot_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ] - }, - "need_deploy": false - }, - "hetero_secureboost_1": { - "module": "HeteroSecureBoost", - "input": { - "data": { - "train_data": [ - "one_hot_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "train" - ] - } - }, - "evaluation_0": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "hetero_lr_0.train" - ] - } - } - }, - "evaluation_1": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "hetero_secureboost_1.train" - ] - } - } - } - } -} diff --git a/examples/advanced/using_cache_simple_conf.json b/examples/advanced/using_cache_simple_conf.json deleted file mode 100644 index 56f1ceccf..000000000 --- a/examples/advanced/using_cache_simple_conf.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "dsl_version": "2", - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "guest": [ - 9999 - ], - "host": [ - 10000 - ], - "arbiter": [ - 10000 - ] - }, - "job_parameters": { - "common": { - "task_parallelism": 1, - "auto_retries": 1, - "computing_partitions": 8 - } - }, - "component_parameters": { - "common": { - "intersect_0": { - "intersect_method": "raw", - "sync_intersect_ids": true, - "only_output_key": false - }, - "hetero_lr_0": { - "penalty": "L2", - "optimizer": "rmsprop", - "alpha": 0.01, - "max_iter": 3, - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - } - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": true, - "label_name": "y", - "label_type": "int", - "output_format": "dense" - }, - "cache_loader_0": { - "job_id": "202108311713595638160", - "component_name": "data_transform_0", - "cache_name": "0" - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": false, - "output_format": "dense" - }, - "cache_loader_0": { - "job_id": "202108311713595638160", - "component_name": "data_transform_0", - "cache_name": "0" - } - } - } - } - } -} diff --git a/examples/advanced/using_cache_simple_dsl.json b/examples/advanced/using_cache_simple_dsl.json deleted file mode 100644 index 12b169955..000000000 --- a/examples/advanced/using_cache_simple_dsl.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "table" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.table" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "data_transform" - ] - }, - "need_deploy": true - }, - "cache_loader_0": { - "module": "CacheLoader", - "output": { - "cache": [ - "cache2" - ] - } - }, - "hetero_feature_binning_0": { - "module": "HeteroFeatureBinning", - "input": { - "data": { - "data": [ - "data_transform_0.train" - ] - }, - "cache": [ - "cache_loader_0.cache2" - ] - }, - "output": { - "data": [ - "train" - ], - "model": [ - "hetero_feature_binning" - ] - } - } - } -} diff --git "a/examples/bfia/fate/component_define/lr\347\256\227\346\263\225\346\217\217\350\277\260.json" "b/examples/bfia/fate/component_define/lr\347\256\227\346\263\225\346\217\217\350\277\260.json" new file mode 100644 index 000000000..c6c0717fc --- /dev/null +++ "b/examples/bfia/fate/component_define/lr\347\256\227\346\263\225\346\217\217\350\277\260.json" @@ -0,0 +1,186 @@ +{ + "componentName": "coordinated_lr", + "title": "lr", + "provider": "fate", + "version": "2.0.0", + "description": "逻辑回归算法", + "roleList": [ + "guest", + "host", + "arbiter" + ], + "desVersion": "2.0.0", + "storageEngine": [ + "s3", + "hdfs", + "eggroll" + ], + "inputParam": [ + { + "name": "learning_rate_scheduler", + "title": "", + "description": "learning rate scheduler", + "type": "string", + "optional": "true", + "defaultValue": "{'method': 'linear', 'scheduler_params': {'start_factor': 1.0}}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "epochs", + "title": "", + "description": "max iteration num", + "type": "integer", + "optional": "true", + "defaultValue": 20, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "batch_size", + "title": "", + "description": "batch size, None means full batch, otherwise should be no less than 10, default None", + "type": "integer", + "optional": "true", + "defaultValue": null, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "optimizer", + "title": "", + "description": "", + "type": "string", + "optional": "true", + "defaultValue": "{'method': 'sgd', 'penalty': 'l2', 'plpah': 1.0, 'optimizer_params': {'lr': 1e-2, 'weight_decay': 0}}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "floating_point_precision", + "title": "", + "description": "floating point precision", + "type": "integer", + "optional": "true", + "defaultValue": 23, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "tol", + "title": "", + "description": "", + "type": "float", + "optional": "true", + "defaultValue": 1e-4, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "early_stop", + "title": "", + "description": "early stopping criterion, choose from {weight_diff, diff, abs, val_metrics}", + "type": "string", + "optional": "true", + "defaultValue": "diff", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "he_param", + "title": "", + "description": "homomorphic encryption param", + "type": "string", + "optional": "true", + "defaultValue": "{'kind': 'paillier', 'key_length': 1024}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "init_param", + "title": "", + "description": "Model param init setting.", + "type": "string", + "optional": "true", + "defaultValue": "{'method': 'random_uniform', 'fit_intercept': true, 'random_state': null}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "threshold", + "title": "", + "description": "predict threshold for binary data", + "type": "float", + "optional": "true", + "defaultValue": 0.5, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + } + ], + "outputData": [ + { + "name": "train_output_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + }, + { + "name": "output_model", + "description": "模型", + "category": "model", + "dataFormat": [ + "json" + ] + }, + { + "name": "metric", + "description": "report", + "category": "report", + "dataFormat": [ + "json" + ] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] + } \ No newline at end of file diff --git "a/examples/bfia/fate/component_define/psi\347\256\227\346\263\225\346\217\217\350\277\260.json" "b/examples/bfia/fate/component_define/psi\347\256\227\346\263\225\346\217\217\350\277\260.json" new file mode 100644 index 000000000..62b047d58 --- /dev/null +++ "b/examples/bfia/fate/component_define/psi\347\256\227\346\263\225\346\217\217\350\277\260.json" @@ -0,0 +1,81 @@ +{ + "componentName": "psi", + "title": "对齐算法", + "provider": "fate", + "version": "2.0.0", + "description": "对齐算法", + "roleList": [ + "guest", + "host" + ], + "desVersion": "2.0.0", + "storageEngine": [ + "s3", + "hdfs", + "eggroll" + ], + "inputParam": [ + { + "name": "protocol", + "title": "protocol", + "description": "protocol", + "type": "string", + "optional": "true", + "defaultValue": "ecdh_psi", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "curve_type", + "title": "curve_type", + "description": "curve_type", + "type": "string", + "optional": "true", + "defaultValue": "curve25519", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [ + { + "name": "input_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + } + ], + "outputData": [ + { + "name": "output_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + }, + { + "name": "metric", + "description": "对齐数", + "category": "report", + "dataFormat": [ + "json" + ] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] + } \ No newline at end of file diff --git "a/examples/bfia/fate/component_define/sbt\347\256\227\346\263\225\346\217\217\350\277\260.json" "b/examples/bfia/fate/component_define/sbt\347\256\227\346\263\225\346\217\217\350\277\260.json" new file mode 100644 index 000000000..527418efa --- /dev/null +++ "b/examples/bfia/fate/component_define/sbt\347\256\227\346\263\225\346\217\217\350\277\260.json" @@ -0,0 +1,245 @@ +{ + "componentName": "hetero_secureboost", + "title": "sbt", + "provider": "fate", + "version": "2.0.0", + "description": "XGBoost算法", + "roleList": [ + "guest", + "host" + ], + "desVersion": "2.0.0", + "storageEngine": [ + "s3", + "hdfs", + "eggroll" + ], + "inputParam": [ + { + "name": "num_trees", + "title": "", + "description": "", + "type": "integer", + "optional": "true", + "defaultValue": 3, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "learning_rate", + "title": "", + "description": "", + "type": "float", + "optional": "true", + "defaultValue": 0.3, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "max_depth", + "title": "", + "description": "max depth of a tree", + "type": "integer", + "optional": "true", + "defaultValue": 3, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "max_bin", + "title": "", + "description": "max bin number of feature binning", + "type": "integer", + "optional": "true", + "defaultValue": 32, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "objective", + "title": "", + "description": "", + "type": "string", + "optional": "true", + "defaultValue": "binary:bce", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "num_class", + "title": "", + "description": "", + "type": "integer", + "optional": "true", + "defaultValue": 2, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "l2", + "title": "", + "description": "L2 regularization", + "type": "float", + "optional": "true", + "defaultValue": 0.1, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_impurity_split", + "title": "", + "description": "min impurity when splitting a tree node", + "type": "float", + "optional": "true", + "defaultValue": 1e-2, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_sample_split", + "title": "", + "description": "min sample to split a tree node", + "type": "integer", + "optional": "true", + "defaultValue": 2, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_leaf_node", + "title": "", + "description": "mininum sample contained in a leaf node", + "type": "integer", + "optional": "true", + "defaultValue": 1, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_child_weight", + "title": "", + "description": "minumum hessian contained in a leaf node", + "type": "integer", + "optional": "true", + "defaultValue": 1, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "gh_pack", + "title": "", + "description": "whether to pack gradient and hessian together", + "type": "bool", + "optional": "true", + "defaultValue": true, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "split_info_pack", + "title": "", + "description": "for host side, whether to pack split info together", + "type": "bool", + "optional": "true", + "defaultValue": true, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "hist_sub", + "title": "", + "description": "whether to use histogram subtraction", + "type": "bool", + "optional": "true", + "defaultValue": true, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "he_param", + "title": "", + "description": "whether to use histogram subtraction", + "type": "string", + "optional": "true", + "defaultValue": "{'kind': 'paillier', 'key_length': 1024}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + } + ], + "outputData": [ + { + "name": "train_data_output", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + }, + { + "name": "train_model_output", + "description": "输出模型", + "category": "model", + "dataFormat": [ + "json" + ] + }, + { + "name": "metric", + "description": "对齐数", + "category": "report", + "dataFormat": [ + "json" + ] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] + } \ No newline at end of file diff --git "a/examples/bfia/fate/component_define/transformer\347\256\227\346\263\225\346\217\217\350\277\260.json" "b/examples/bfia/fate/component_define/transformer\347\256\227\346\263\225\346\217\217\350\277\260.json" new file mode 100644 index 000000000..f24c8abae --- /dev/null +++ "b/examples/bfia/fate/component_define/transformer\347\256\227\346\263\225\346\217\217\350\277\260.json" @@ -0,0 +1,93 @@ +{ + "componentName": "dataframe_transformer", + "title": "dataframe transformer", + "provider": "fate", + "version": "2.0.0", + "description": "数据转换", + "roleList": [ + "guest", + "host" + ], + "desVersion": "2.0.0", + "storageEngine": [ + "s3", + "hdfs", + "eggroll" + ], + "inputParam": [ + { + "name": "namespace", + "title": "", + "description": "namespace", + "type": "string", + "optional": "true", + "defaultValue": null, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "name", + "title": "", + "description": "name", + "type": "string", + "optional": "true", + "defaultValue": null, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "site_name", + "title": "", + "description": "site name", + "type": "string", + "optional": "true", + "defaultValue": null, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [ + { + "name": "table", + "description": "upload数据集", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + } + ], + "outputData": [ + { + "name": "dataframe_output", + "description": "dataframe数据集", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + }, + { + "name": "metric", + "description": "metric", + "category": "report", + "dataFormat": [ + "json" + ] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] + } \ No newline at end of file diff --git a/examples/bfia/fate/job/dataframe_transformer.yaml b/examples/bfia/fate/job/dataframe_transformer.yaml new file mode 100644 index 000000000..8150aaea0 --- /dev/null +++ b/examples/bfia/fate/job/dataframe_transformer.yaml @@ -0,0 +1,44 @@ +dag: + initiator: + - "guest" + - "JG0100001100000010" + conf: + extra: + initiator: + role: guest + party_id: JG0100001100000010 + parties: + - party_id: + - JG0100001100000010 + role: guest + stage: default + tasks: + transformer_0: + parameters: + name: breast_hetero_guest + namespace: experiment + site_name: null + conf: + provider: "fate" + version: "2.0.0" + inputs: + data: + table: + data_warehouse: + dataset_id: upload#guest + outputs: + data: + dataframe_output: + output_artifact_key_alias: dataframe_output + output_artifact_type_alias: dataset + metric: + metric: + output_artifact_key_alias: metric + output_artifact_type_alias: report + component_ref: dataframe_transformer + parties: + - party_id: + - JG0100001100000010 + role: guest +schema_version: 2.0.0 +kind: bfia diff --git a/examples/bfia/fate/job/psi_lr.yaml b/examples/bfia/fate/job/psi_lr.yaml new file mode 100644 index 000000000..5c7e00768 --- /dev/null +++ b/examples/bfia/fate/job/psi_lr.yaml @@ -0,0 +1,117 @@ +dag: + conf: + extra: + flow_id: '' + initiator: + party_id: JG0100001100000010 + role: guest + old_job_id: '' + parties: + - party_id: + - JG0100001100000010 + role: guest + - party_id: + - JG0100001100000010 + role: host + - party_id: + - JG0100001100000010 + role: arbiter + party_tasks: + guest_JG0100001100000010: + conf: + resources: + cpu: -1 + disk: -1 + memory: -1 + parties: + - party_id: + - JG0100001100000010 + role: guest + host_JG0100001100000010: + conf: + resources: + cpu: -1 + disk: -1 + memory: -1 + parties: + - party_id: + - JG0100001100000010 + role: host + tasks: + lr_0: + component_ref: coordinated_lr + conf: + provider: fate + version: 2.0.0 + dependent_tasks: + - psi_0 + inputs: + data: + train_data: + task_output_artifact: + output_artifact_key: output_data + output_artifact_type_alias: dataset + producer_task: psi_0 + outputs: + data: + train_output_data: + output_artifact_key_alias: train_output_data + output_artifact_type_alias: dataset + metric: + metric: + output_artifact_key_alias: metric + output_artifact_type_alias: report + model: + output_model: + output_artifact_key_alias: output_model + output_artifact_type_alias: model + parameters: {} + parties: + - party_id: + - JG0100001100000010 + role: guest + - party_id: + - JG0100001100000010 + role: host + - party_id: + - JG0100001100000010 + role: arbiter + psi_0: + component_ref: psi + conf: + provider: fate + version: 2.0.0 + dependent_tasks: [] + inputs: + data: + input_data: + data_warehouse: + - dataset_id: test#guest + parties: + - party_id: + - JG0100001100000010 + role: guest + - dataset_id: test#host + parties: + - party_id: + - JG0100001100000010 + role: host + outputs: + data: + output_data: + output_artifact_key_alias: output_data + output_artifact_type_alias: dataset + metric: + metric: + output_artifact_key_alias: metric + output_artifact_type_alias: report + parameters: {} + parties: + - party_id: + - JG0100001100000010 + role: guest + - party_id: + - JG0100001100000010 + role: host +kind: bfia +schema_version: v1 diff --git a/examples/bfia/fate/job/psi_sbt.yaml b/examples/bfia/fate/job/psi_sbt.yaml new file mode 100644 index 000000000..4b35cc209 --- /dev/null +++ b/examples/bfia/fate/job/psi_sbt.yaml @@ -0,0 +1,111 @@ +dag: + conf: + extra: + flow_id: '' + initiator: + party_id: JG0100001100000010 + role: guest + old_job_id: '' + parties: + - party_id: + - JG0100001100000010 + role: guest + - party_id: + - JG0100001100000010 + role: host + party_tasks: + guest_JG0100001100000010: + conf: + resources: + cpu: -1 + disk: -1 + memory: -1 + parties: + - party_id: + - JG0100001100000010 + role: guest + host_JG0100001100000010: + conf: + resources: + cpu: -1 + disk: -1 + memory: -1 + parties: + - party_id: + - JG0100001100000010 + role: host + tasks: + psi_0: + component_ref: psi + conf: + provider: fate + version: 2.0.0 + dependent_tasks: [] + inputs: + data: + input_data: + data_warehouse: + - dataset_id: test#guest + parties: + - party_id: + - JG0100001100000010 + role: guest + - dataset_id: test#host + parties: + - party_id: + - JG0100001100000010 + role: host + outputs: + data: + output_data: + output_artifact_key_alias: output_data + output_artifact_type_alias: dataset + metric: + metric: + output_artifact_key_alias: metric + output_artifact_type_alias: report + parameters: {} + parties: + - party_id: + - JG0100001100000010 + role: guest + - party_id: + - JG0100001100000010 + role: host + sbt_0: + component_ref: hetero_secureboost + conf: + provider: fate + version: 2.0.0 + dependent_tasks: + - psi_0 + inputs: + data: + train_data: + task_output_artifact: + output_artifact_key: output_data + output_artifact_type_alias: dataset + producer_task: psi_0 + outputs: + data: + train_data_output: + output_artifact_key_alias: train_data_output + output_artifact_type_alias: dataset + metric: + metric: + output_artifact_key_alias: metric + output_artifact_type_alias: report + model: + train_model_output: + output_artifact_key_alias: train_model_output + output_artifact_type_alias: model + parameters: {} + parties: + - party_id: + - JG0100001100000010 + role: guest + - party_id: + - JG0100001100000010 + role: host +kind: bfia +schema_version: v1 diff --git a/examples/bfia/fate/pipeline/test_dataframe_transformer.py b/examples/bfia/fate/pipeline/test_dataframe_transformer.py new file mode 100644 index 000000000..4f59b01fa --- /dev/null +++ b/examples/bfia/fate/pipeline/test_dataframe_transformer.py @@ -0,0 +1,48 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from fate_client.pipeline import FateFlowPipeline +from fate_client.pipeline.components.fate import DataFrameTransformer +from fate_client.pipeline.interface.channel import DataWarehouseChannel + + +def main(): + guest_party_id = "JG0100001100000010" + + pipeline = FateFlowPipeline().set_parties(guest=guest_party_id) + transformer_0 = DataFrameTransformer( + "transformer_0", + namespace="test", + name="guest", + table=DataWarehouseChannel( + dataset_id="upload#guest" + ) + ) + pipeline.set_site_role("guest") + pipeline.set_site_party_id(guest_party_id) + + pipeline.add_tasks([transformer_0]) + pipeline.protocol_kind = "bfia" + pipeline.conf.set( + "extra", + dict(initiator={'party_id': 'JG0100001100000010', 'role': 'guest'}) + ) + pipeline.compile() + # print(pipeline.get_dag()) + pipeline.fit() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/bfia/fate/pipeline/test_lr.py b/examples/bfia/fate/pipeline/test_lr.py new file mode 100644 index 000000000..294d8310b --- /dev/null +++ b/examples/bfia/fate/pipeline/test_lr.py @@ -0,0 +1,58 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from fate_client.pipeline import FateFlowPipeline +from fate_client.pipeline.components.fate import CoordinatedLR, PSI, Reader +from fate_client.pipeline.components.fate import Evaluation +from fate_client.pipeline.interface.channel import DataWarehouseChannel + + +def main(): + guest = "JG0100001100000010" + host = "JG0100001100000010" + arbiter = "JG0100001100000010" + pipeline = FateFlowPipeline().set_parties(guest=guest, host=host, arbiter=arbiter) + pipeline.set_site_role("guest") + pipeline.set_site_party_id(guest) + + psi_0 = PSI("psi_0", + input_data=[DataWarehouseChannel(dataset_id="experiment#breast_hetero_guest", parties=dict(guest=guest)), + DataWarehouseChannel(dataset_id="experiment#breast_hetero_host", parties=dict(host=host))]) + lr_0 = CoordinatedLR("lr_0", + epochs=10, + batch_size=300, + optimizer={"method": "SGD", "optimizer_params": {"lr": 0.1}, "penalty": "l2", "alpha": 0.001}, + init_param={"fit_intercept": True, "method": "zeros"}, + train_data=psi_0.outputs["output_data"], + learning_rate_scheduler={"method": "linear", "scheduler_params": {"start_factor": 0.7, + "total_iters": 100}}) + + pipeline.add_tasks([psi_0, lr_0]) + + pipeline.protocol_kind = "bfia" + pipeline.conf.set( + "extra", + dict(initiator={'party_id': guest, 'role': 'guest'}) + ) + pipeline.guest.conf.set("resources", dict(cpu=-1, disk=-1, memory=-1)) + pipeline.hosts[0].conf.set("resources", dict(cpu=-1, disk=-1, memory=-1)) + pipeline.compile() + pipeline.fit() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/bfia/fate/pipeline/test_sbt.py b/examples/bfia/fate/pipeline/test_sbt.py new file mode 100644 index 000000000..1e55b063c --- /dev/null +++ b/examples/bfia/fate/pipeline/test_sbt.py @@ -0,0 +1,58 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from fate_client.pipeline import FateFlowPipeline +from fate_client.pipeline.components.fate import HeteroSecureBoost, PSI, Reader +from fate_client.pipeline.components.fate import Evaluation +from fate_client.pipeline.interface.channel import DataWarehouseChannel + + +def main(): + guest = "JG0100001100000010" + host = "JG0100001100000010" + + pipeline = FateFlowPipeline().set_parties(guest=guest, host=host) + + pipeline.set_site_role("guest") + pipeline.set_site_party_id(guest) + + psi_0 = PSI("psi_0", + input_data=[DataWarehouseChannel(dataset_id="experiment#breast_hetero_guest", parties=dict(guest=guest)), + DataWarehouseChannel(dataset_id="experiment#breast_hetero_host", parties=dict(host=host))]) + hetero_sbt_0 = HeteroSecureBoost( + "hetero_sbt_0", + train_data=psi_0.outputs["output_data"], + num_trees=1, + max_depth=3, + ) + + + pipeline.add_tasks([psi_0, hetero_sbt_0]) + + pipeline.protocol_kind = "bfia" + pipeline.conf.set( + "extra", + dict(initiator={'party_id': guest, 'role': 'guest'}) + ) + pipeline.guest.conf.set("resources", dict(cpu=-1, disk=-1, memory=-1)) + pipeline.hosts[0].conf.set("resources", dict(cpu=-1, disk=-1, memory=-1)) + pipeline.compile() + pipeline.fit() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/bfia/fate/register/fate_components.json b/examples/bfia/fate/register/fate_components.json new file mode 100644 index 000000000..0ae9698aa --- /dev/null +++ b/examples/bfia/fate/register/fate_components.json @@ -0,0 +1,617 @@ +{ + "name": "fate", + "device": "docker", + "version": "2.0.0", + "metadata": { + "base_url": "", + "image": "federatedai/fate:2.0.0" + }, + "protocol": "bfia", + "components_description": { + "dataframe_transformer": { + "componentName": "dataframe_transformer", + "title": "dataframe transformer", + "provider": "fate", + "version": "2.0.0", + "description": "数据转换", + "roleList": [ + "guest", + "host" + ], + "desVersion": "2.0.0", + "storageEngine": [ + "s3", + "hdfs", + "eggroll" + ], + "inputParam": [ + { + "name": "namespace", + "title": "", + "description": "namespace", + "type": "string", + "optional": "true", + "defaultValue": null, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "name", + "title": "", + "description": "name", + "type": "string", + "optional": "true", + "defaultValue": null, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "site_name", + "title": "", + "description": "site name", + "type": "string", + "optional": "true", + "defaultValue": null, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [ + { + "name": "table", + "description": "upload数据集", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + } + ], + "outputData": [ + { + "name": "dataframe_output", + "description": "dataframe数据集", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + }, + { + "name": "metric", + "description": "metric", + "category": "report", + "dataFormat": [ + "json" + ] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] + }, + "psi": { + "componentName": "psi", + "title": "对齐算法", + "provider": "fate", + "version": "2.0.0", + "description": "对齐算法", + "roleList": [ + "guest", + "host" + ], + "desVersion": "2.0.0", + "storageEngine": [ + "s3", + "hdfs", + "eggroll" + ], + "inputParam": [ + { + "name": "protocol", + "title": "protocol", + "description": "protocol", + "type": "string", + "optional": "true", + "defaultValue": "ecdh_psi", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "curve_type", + "title": "curve_type", + "description": "curve_type", + "type": "string", + "optional": "true", + "defaultValue": "curve25519", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [ + { + "name": "input_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + } + ], + "outputData": [ + { + "name": "output_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + }, + { + "name": "metric", + "description": "对齐数", + "category": "report", + "dataFormat": [ + "json" + ] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] + }, + "coordinated_lr": { + "componentName": "coordinated_lr", + "title": "lr", + "provider": "fate", + "version": "2.0.0", + "description": "逻辑回归算法", + "roleList": [ + "guest", + "host", + "arbiter" + ], + "desVersion": "2.0.0", + "storageEngine": [ + "s3", + "hdfs", + "eggroll" + ], + "inputParam": [ + { + "name": "learning_rate_scheduler", + "title": "", + "description": "learning rate scheduler", + "type": "string", + "optional": "true", + "defaultValue": "{'method': 'linear', 'scheduler_params': {'start_factor': 1.0}}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "epochs", + "title": "", + "description": "max iteration num", + "type": "integer", + "optional": "true", + "defaultValue": 20, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "batch_size", + "title": "", + "description": "batch size, None means full batch, otherwise should be no less than 10, default None", + "type": "integer", + "optional": "true", + "defaultValue": null, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "optimizer", + "title": "", + "description": "", + "type": "string", + "optional": "true", + "defaultValue": "{'method': 'sgd', 'penalty': 'l2', 'plpah': 1.0, 'optimizer_params': {'lr': 1e-2, 'weight_decay': 0}}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "floating_point_precision", + "title": "", + "description": "floating point precision", + "type": "integer", + "optional": "true", + "defaultValue": 23, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "tol", + "title": "", + "description": "", + "type": "float", + "optional": "true", + "defaultValue": 1e-4, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "early_stop", + "title": "", + "description": "early stopping criterion, choose from {weight_diff, diff, abs, val_metrics}", + "type": "string", + "optional": "true", + "defaultValue": "diff", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "he_param", + "title": "", + "description": "homomorphic encryption param", + "type": "string", + "optional": "true", + "defaultValue": "{'kind': 'paillier', 'key_length': 1024}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "init_param", + "title": "", + "description": "Model param init setting.", + "type": "string", + "optional": "true", + "defaultValue": "{'method': 'random_uniform', 'fit_intercept': true, 'random_state': null}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "threshold", + "title": "", + "description": "predict threshold for binary data", + "type": "float", + "optional": "true", + "defaultValue": 0.5, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + } + ], + "outputData": [ + { + "name": "train_output_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + }, + { + "name": "output_model", + "description": "模型", + "category": "model", + "dataFormat": [ + "json" + ] + }, + { + "name": "metric", + "description": "report", + "category": "report", + "dataFormat": [ + "json" + ] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] + }, + "hetero_secureboost": { + "componentName": "hetero_secureboost", + "title": "sbt", + "provider": "fate", + "version": "2.0.0", + "description": "XGBoost算法", + "roleList": [ + "guest", + "host" + ], + "desVersion": "2.0.0", + "storageEngine": [ + "s3", + "hdfs", + "eggroll" + ], + "inputParam": [ + { + "name": "num_trees", + "title": "", + "description": "", + "type": "integer", + "optional": "true", + "defaultValue": 3, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "learning_rate", + "title": "", + "description": "", + "type": "float", + "optional": "true", + "defaultValue": 0.3, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "max_depth", + "title": "", + "description": "max depth of a tree", + "type": "integer", + "optional": "true", + "defaultValue": 3, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "max_bin", + "title": "", + "description": "max bin number of feature binning", + "type": "integer", + "optional": "true", + "defaultValue": 32, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "objective", + "title": "", + "description": "", + "type": "string", + "optional": "true", + "defaultValue": "binary:bce", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "num_class", + "title": "", + "description": "", + "type": "integer", + "optional": "true", + "defaultValue": 2, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "l2", + "title": "", + "description": "L2 regularization", + "type": "float", + "optional": "true", + "defaultValue": 0.1, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_impurity_split", + "title": "", + "description": "min impurity when splitting a tree node", + "type": "float", + "optional": "true", + "defaultValue": 1e-2, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_sample_split", + "title": "", + "description": "min sample to split a tree node", + "type": "integer", + "optional": "true", + "defaultValue": 2, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_leaf_node", + "title": "", + "description": "mininum sample contained in a leaf node", + "type": "integer", + "optional": "true", + "defaultValue": 1, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_child_weight", + "title": "", + "description": "minumum hessian contained in a leaf node", + "type": "integer", + "optional": "true", + "defaultValue": 1, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "gh_pack", + "title": "", + "description": "whether to pack gradient and hessian together", + "type": "bool", + "optional": "true", + "defaultValue": true, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "split_info_pack", + "title": "", + "description": "for host side, whether to pack split info together", + "type": "bool", + "optional": "true", + "defaultValue": true, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "hist_sub", + "title": "", + "description": "whether to use histogram subtraction", + "type": "bool", + "optional": "true", + "defaultValue": true, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "he_param", + "title": "", + "description": "whether to use histogram subtraction", + "type": "string", + "optional": "true", + "defaultValue": "{'kind': 'paillier', 'key_length': 1024}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + } + ], + "outputData": [ + { + "name": "train_data_output", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + }, + { + "name": "train_model_output", + "description": "输出模型", + "category": "model", + "dataFormat": [ + "json" + ] + }, + { + "name": "metric", + "description": "对齐数", + "category": "report", + "dataFormat": [ + "json" + ] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] + } + } +} \ No newline at end of file diff --git a/examples/bfia/fate/upload/upload_guest.py b/examples/bfia/fate/upload/upload_guest.py new file mode 100644 index 000000000..a6f7bf0bd --- /dev/null +++ b/examples/bfia/fate/upload/upload_guest.py @@ -0,0 +1,44 @@ +import os +import tempfile + +from fate_flow.adapter.bfia.container.wraps.wraps import DataIo +from fate_flow.components.components.upload import Upload, UploadParam +from fate_flow.entity.spec.dag import Metadata + + +def upload_data(s3_address, namespace, name, file, meta, head=True, partitions=16, extend_sid=True, storage_engine="standalone"): + upload_object = Upload() + params = { + 'name': name, + 'namespace': namespace, + 'file': file, + 'storage_engine': storage_engine, + 'head': head, + 'partitions': partitions, + 'extend_sid': extend_sid, + 'meta': meta + } + params = UploadParam(**params) + + with tempfile.TemporaryDirectory() as data_home: + os.environ["STANDALONE_DATA_HOME"] = data_home + data_meta = upload_object.run(params).get("data_meta") + + metadata = Metadata(metadata=dict(options=dict(partitions=partitions), schema=data_meta)) + data_path = os.path.join(data_home, namespace, name) + engine = DataIo(s3_address) + engine.upload_to_s3(data_path, name=name, namespace=namespace, metadata=metadata.dict()) + + +if __name__ == "__main__": + s3_address = "s3://127.0.0.1:9000?username=admin&password=12345678" + namespace = "upload" + name = "guest" + file = 'examples/data/breast_hetero_guest.csv' + + meta = { + "delimiter": ",", + "label_name": "y", + "match_id_name": "id" + } + upload_data(s3_address=s3_address, namespace=namespace, name=name, file=file, meta=meta) diff --git a/examples/bfia/fate/upload/upload_host.py b/examples/bfia/fate/upload/upload_host.py new file mode 100644 index 000000000..9b4aa8677 --- /dev/null +++ b/examples/bfia/fate/upload/upload_host.py @@ -0,0 +1,43 @@ +import os +import tempfile + +from fate_flow.adapter.bfia.container.wraps.wraps import DataIo +from fate_flow.components.components.upload import Upload, UploadParam +from fate_flow.entity.spec.dag import Metadata + + +def upload_data(s3_address, namespace, name, file, meta, head=True, partitions=16, extend_sid=True, storage_engine="standalone"): + upload_object = Upload() + params = { + 'name': name, + 'namespace': namespace, + 'file': file, + 'storage_engine': storage_engine, + 'head': head, + 'partitions': partitions, + 'extend_sid': extend_sid, + 'meta': meta + } + params = UploadParam(**params) + + with tempfile.TemporaryDirectory() as data_home: + os.environ["STANDALONE_DATA_HOME"] = data_home + data_meta = upload_object.run(params).get("data_meta") + + metadata = Metadata(metadata=dict(options=dict(partitions=partitions), schema=data_meta)) + data_path = os.path.join(data_home, namespace, name) + engine = DataIo(s3_address) + engine.upload_to_s3(data_path, name=name, namespace=namespace, metadata=metadata.dict()) + + +if __name__ == "__main__": + s3_address = "s3://127.0.0.1:9000?username=admin&password=12345678" + namespace = "upload" + name = "host" + file = 'examples/data/breast_hetero_host.csv' + + meta = { + "delimiter": ",", + "match_id_name": "id" + } + upload_data(s3_address=s3_address, namespace=namespace, name=name, file=file, meta=meta) diff --git "a/examples/bfia/unionpay/component_define/lr\347\256\227\346\263\225\346\217\217\350\277\260.json" "b/examples/bfia/unionpay/component_define/lr\347\256\227\346\263\225\346\217\217\350\277\260.json" new file mode 100644 index 000000000..77055cc58 --- /dev/null +++ "b/examples/bfia/unionpay/component_define/lr\347\256\227\346\263\225\346\217\217\350\277\260.json" @@ -0,0 +1,236 @@ +{ + "componentName": "HeteroLR", + "title": "纵向逻辑回归算法", + "provider": "unionpay", + "version": "2.0.0", + "description": "纵向逻辑回归算法", + "roleList": ["guest", "host", "arbiter"], + "desVersion": "1.2.0", + "storageEngine": ["s3","hdfs"], + "inputParam": [ + { + "name": "id", + "title": "id列", + "description": "id字段名", + "type": "string", + "optional": "true", + "defaultValue": "x1", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "label", + "title": "标签", + "description": "label字段名", + "type": "string", + "optional": "true", + "defaultValue": "y", + "validator": "regular-正则项", + "dependsOn": [], + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "penalty", + "title": "正则项", + "description": "正则项", + "type": "string", + "bindingData": [ + { + "label": "L1正则", + "value": "L1" + }, + { + "label": "L2正则", + "value": "L2" + } + ], + "optional": "true", + "defaultValue": "L2", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "tol", + "title": "最小损失值", + "description": "最小损失值", + "type": "float", + "optional": "true", + "defaultValue": "0.0001", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "alpha", + "title": "惩罚因子", + "description": "惩罚因子", + "type": "float", + "optional": "true", + "defaultValue": "0.01", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "optimizer", + "title": "优化方法", + "description": "优化方法", + "type": "string", + "bindingData": [ + { + "label": "rmsprop", + "value": "rmsprop" + }, + { + "label": "sgd", + "value": "sgd" + }, + { + "label": "adam", + "value": "adam" + }, + { + "label": "sqn", + "value": "sqn" + }, + { + "label": "adagrad", + "value": "adagrad" + } + ], + "optional": "true", + "defaultValue": "rmsprop", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "batch_size", + "title": "批量梯度下降样本量", + "description": "每轮迭代抽取数据计算梯度的size", + "type": "integer", + "bindingData": [ + { + "label": "all", + "value": "all" + }, + { + "label": "2048", + "value": "2048" + }, + { + "label": "4096", + "value": "4096" + }, + { + "label": "8192", + "value": "8192" + } + ], + "optional": "true", + "defaultValue": "2048", + "validator": "(0,1000)", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "learning_rate", + "title": "学习率", + "description": "学习率", + "type": "float", + "optional": "true", + "defaultValue": "0.15", + "validator": "regular-正则项", + "dependsOn": ["optimizer.sgd", "optimizer.adam"], + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "init_param", + "title": "初始化方式", + "description": "初始化方式", + "type": "string", + "optional": "true", + "defaultValue": "zeros", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "max_iter", + "title": "迭代次数", + "description": "迭代次数", + "type": "integer", + "optional": "true", + "defaultValue": "30", + "validator": "(0,1000)", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "early_stop", + "title": "早停策略", + "description": "早停策略", + "type": "string", + "bindingData": [ + { + "label": "weight_diff", + "value": "weight_diff" + }, + { + "label": "diff", + "value": "diff" + } + ], + "optional": "true", + "defaultValue": "weight_diff", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + } + ], + "inputData": [{ + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }], + "outputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }, + { + "name": "model", + "description": "模型文件", + "category": "model", + "dataFormat": ["csv"] + }, + { + "name": "train-loss", + "description": "loss值", + "category": "metric", + "dataFormat": ["json"] + } + ], + "result": [{ + "resultCode": "4444", + "resultMessage": "算法执行失败" + }] +} \ No newline at end of file diff --git "a/examples/bfia/unionpay/component_define/psi\347\256\227\346\263\225\346\217\217\350\277\260.json" "b/examples/bfia/unionpay/component_define/psi\347\256\227\346\263\225\346\217\217\350\277\260.json" new file mode 100644 index 000000000..a940adafc --- /dev/null +++ "b/examples/bfia/unionpay/component_define/psi\347\256\227\346\263\225\346\217\217\350\277\260.json" @@ -0,0 +1,190 @@ +{ + "componentName": "Intersection", + "title": "对齐算法", + "provider": "unionpay", + "version": "2.0.0", + "description": "对齐算法", + "roleList": ["guest", "host"], + "desVersion": "1.3.0", + "storageEngine": ["s3","hdfs"], + "inputParam": [ + { + "name": "id", + "title": "id列", + "description": "id字段名", + "type": "string", + "optional": "true", + "defaultValue": "x1", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "intersect_method", + "title": "对齐方式", + "description": "对齐方式", + "type": "string", + "optional": "true", + "defaultValue": "raw", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "sync_intersect_ids", + "title": "同步对齐id", + "description": "同步对齐id", + "type": "boolean", + "optional": "true", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "only_output_key", + "title": "仅输出id", + "description": "仅输出id", + "type": "boolean", + "optional": "true", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "use_hash", + "title": "是否使用哈希", + "description": "是否使用哈希", + "type": "string", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "optional": "true", + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "hash_method", + "title": "哈希方法", + "description": "哈希方法", + "type": "string", + "optional": "true", + "defaultValue": "sha256", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "salt", + "title": "salt", + "description": "salt", + "type": "string", + "optional": "true", + "defaultValue": "12345", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "base64", + "title": "选择base64", + "description": "是否选择base64方式", + "type": "string", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "optional": "true", + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "join_role", + "title": "参与角色", + "description": "参与角色", + "type": "string", + "optional": "true", + "defaultValue": "host", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [{ + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv","yaml"] + }], + "outputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }, + { + "name": "train-intersection", + "description": "对齐数", + "category": "report", + "dataFormat": ["csv"] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] +} \ No newline at end of file diff --git "a/examples/bfia/unionpay/component_define/sbt\347\256\227\346\263\225\346\217\217\350\277\260.json" "b/examples/bfia/unionpay/component_define/sbt\347\256\227\346\263\225\346\217\217\350\277\260.json" new file mode 100644 index 000000000..2a38dfb55 --- /dev/null +++ "b/examples/bfia/unionpay/component_define/sbt\347\256\227\346\263\225\346\217\217\350\277\260.json" @@ -0,0 +1,230 @@ +{ + "componentName": "HeteroSecureBoost", + "title": "XGBoost算法", + "provider": "unionpay", + "version": "2.0.0", + "description": "XGBoost算法", + "roleList": ["guest", "host"], + "desVersion": "1.3.0", + "storageEngine": ["s3","hdfs"], + "inputParam": [ + { + "name": "objective_param", + "title": "目标参数", + "description": "目标参数", + "type": "string", + "optional": "true", + "bindingData": [ + { + "label": "cross_entropy", + "value": "cross_entropy" + } + ], + "defaultValue": "cross_entropy", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "learning_rate", + "title": "学习率", + "description": "学习率", + "type": "float", + "optional": "true", + "defaultValue": "0.15", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "num_trees", + "title": "树个数", + "description": "树个数", + "type": "integer", + "optional": "true", + "defaultValue": "5", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "subsample_feature_rate", + "title": "子样本率", + "description": "子样本率", + "type": "integer", + "optional": "true", + "defaultValue": "1", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "n_iter_no_change", + "title": "n轮无变化", + "description": "n轮无变化", + "type": "boolean", + "optional": "true", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "tol", + "title": "停止容忍度", + "description": "停止容忍度", + "type": "float", + "optional": "true", + "defaultValue": "0.0001", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "bin_num", + "title": "分位数", + "description": "分位数", + "type": "integer", + "optional": "true", + "defaultValue": "32", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "predict_param", + "title": "预测参数", + "description": "预测参数", + "type": "string", + "optional": "true", + "defaultValue": "{\"threshold\": 0.5}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "cv_param", + "title": "cv参数", + "description": "cv参数", + "type": "string", + "optional": "true", + "defaultValue": "{\"n_splits\": 5, \"shuffle\": false, \"random_seed\": 103, \"need_cv\": false}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "metrics", + "title": "计算指标", + "description": "计算指标", + "type": "string", + "optional": "true", + "defaultValue": "[\"auc\", \"ks\"]", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "early_stop", + "title": "早停策略", + "description": "早停策略", + "type": "string", + "optional": "true", + "bindingData": [ + { + "label": "weight_diff", + "value": "weight_diff" + }, + { + "label": "diff", + "value": "diff" + } + ], + "defaultValue": "weight_diff", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "tree_param", + "title": "树参数", + "description": "树参数", + "type": "string", + "optional": "true", + "defaultValue": "{\"max_depth\": 3}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [{ + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }], + "outputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }, + { + "name": "model", + "description": "模型文件", + "category": "model", + "dataFormat": ["csv"] + }, + { + "name": "train-loss", + "description": "loss值", + "category": "report", + "dataFormat": ["json"] + }, + { + "name": "train-auc", + "description": "auc ks值", + "category": "report", + "dataFormat": ["json"] + }, + { + "name": "train-ks", + "description": "ks曲线值", + "category": "report", + "dataFormat": ["json"] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] +} \ No newline at end of file diff --git a/examples/bfia/unionpay/job/psi_lr.yaml b/examples/bfia/unionpay/job/psi_lr.yaml new file mode 100644 index 000000000..bbb894540 --- /dev/null +++ b/examples/bfia/unionpay/job/psi_lr.yaml @@ -0,0 +1,95 @@ +dag: + conf: + extra: + initiator: {party_id: JG0100001100000010, role: guest} + parties: + - party_id: [JG0100001100000010] + role: guest + - party_id: [JG0100001100000010] + role: host + - party_id: [JG0100001100000010] + role: arbiter + party_tasks: + guest_JG0100001100000010: + conf: + resources: {cpu: -1, disk: -1, memory: -1} + parties: + - party_id: [JG0100001100000010] + role: guest + host_JG0100001100000010: + conf: + resources: {cpu: -1, disk: -1, memory: -1} + parties: + - party_id: [JG0100001100000010] + role: host + stage: train + tasks: + hetero_logistic_regression_1: + component_ref: HeteroLR + conf: {provider: unionpay, version: 2.0.0} + dependent_tasks: [intersect_rsa_1] + inputs: + data: + train_data: + task_output_artifact: + output_artifact_key: train_data + parties: + - party_id: [JG0100001100000010] + role: guest + - party_id: [JG0100001100000010] + role: host + - party_id: [JG0100001100000010] + role: arbiter + producer_task: intersect_rsa_1 + outputs: + data: + train_data: {output_artifact_key_alias: train_data, output_artifact_type_alias: data} + metric: + train-loss: {output_artifact_key_alias: train-loss, output_artifact_type_alias: metric} + model: + model: {output_artifact_key_alias: model, output_artifact_type_alias: model} + parameters: + alpha: 0.01 + batch_size: -1 + early_stop: diff + id: id + init_param: {init_method: zeros} + label: y + learning_rate: 0.15 + max_iter: 2 + optimizer: nesterov_momentum_sgd + penalty: L2 + tol: 0.0001 + intersect_rsa_1: + component_ref: Intersection + conf: {provider: unionpay, version: 2.0.0} + inputs: + data: + train_data: + data_warehouse: + - dataset_id: testspace#test_guest + parties: + - party_id: [JG0100001100000010] + role: guest + - dataset_id: testspace#test_host + parties: + - party_id: [JG0100001100000010] + role: host + outputs: + data: + train_data: {output_artifact_key_alias: train_data, output_artifact_type_alias: data} + metric: + train-intersection: {output_artifact_key_alias: train-intersection, output_artifact_type_alias: metric} + parameters: + id: id + intersect_method: rsa + only_output_key: false + rsa_params: {final_hash_method: sha256, hash_method: sha256, key_length: 2048} + sync_intersect_ids: true + parties: + - party_id: [JG0100001100000010] + role: guest + - party_id: [JG0100001100000010] + role: host +kind: bfia +schema_version: 2.0.0.beta \ No newline at end of file diff --git a/examples/bfia/unionpay/job/psi_sbt.yaml b/examples/bfia/unionpay/job/psi_sbt.yaml new file mode 100644 index 000000000..1f167cebb --- /dev/null +++ b/examples/bfia/unionpay/job/psi_sbt.yaml @@ -0,0 +1,91 @@ +dag: + conf: + extra: + initiator: {party_id: JG0100001100000010, role: guest} + parties: + - party_id: [JG0100001100000010] + role: guest + - party_id: [JG0100001100000010] + role: host + party_tasks: + guest_JG0100001100000010: + conf: + resources: {cpu: -1, disk: -1, memory: -1} + parties: + - party_id: [JG0100001100000010] + role: guest + host_JG0100001100000010: + conf: + resources: {cpu: -1, disk: -1, memory: -1} + parties: + - party_id: [JG0100001100000010] + role: host + stage: train + tasks: + hetero_secureboost_1: + component_ref: HeteroSecureBoost + conf: {provider: unionpay, version: 2.0.0} + dependent_tasks: [intersect_rsa_1] + inputs: + data: + train_data: + task_output_artifact: + output_artifact_key: train_data + parties: + - party_id: [JG0100001100000010] + role: guest + - party_id: [JG0100001100000010] + role: host + producer_task: intersect_rsa_1 + outputs: + data: + train_data: {output_artifact_key_alias: train_data, output_artifact_type_alias: data} + metric: + train-auc: {output_artifact_key_alias: train-auc, output_artifact_type_alias: metric} + train-ks: {output_artifact_key_alias: train-ks, output_artifact_type_alias: metric} + train-loss: {output_artifact_key_alias: train-loss, output_artifact_type_alias: metric} + model: + model: {output_artifact_key_alias: model, output_artifact_type_alias: model} + parameters: + bin_num: 32 + cv_param: {n_splits: 5, need_cv: false, random_seed: 103, shuffle: false} + id: id + label: y + learning_rate: 0.5 + metrics: [auc, ks] + n_iter_no_change: true + num_trees: 2 + objective_param: {objective: cross_entropy} + predict_param: {threshold: 0.5} + subsample_feature_rate: 1 + tol: 0.0001 + tree_param: {max_depth: 5} + intersect_rsa_1: + component_ref: Intersection + conf: {provider: unionpay, version: 2.0.0} + inputs: + data: + train_data: + data_warehouse: + - dataset_id: testspace#test_guest + parties: + - party_id: [JG0100001100000010] + role: guest + - dataset_id: testspace#test_host + parties: + - party_id: [JG0100001100000010] + role: host + outputs: + data: + train_data: {output_artifact_key_alias: train_data, output_artifact_type_alias: data} + metric: + train-intersection: {output_artifact_key_alias: train-intersection, output_artifact_type_alias: metric} + parameters: + id: id + intersect_method: rsa + only_output_key: false + rsa_params: {final_hash_method: sha256, hash_method: sha256, key_length: 2048} + sync_intersect_ids: true +kind: bfia +schema_version: 2.0.0.beta + diff --git a/examples/bfia/unionpay/pipeline/test_unionpay_lr.py b/examples/bfia/unionpay/pipeline/test_unionpay_lr.py new file mode 100644 index 000000000..6bd0f58bf --- /dev/null +++ b/examples/bfia/unionpay/pipeline/test_unionpay_lr.py @@ -0,0 +1,75 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_client.pipeline import FateFlowPipeline +from fate_client.pipeline.adapters.bfia.components.unionpay.intersection import Intersection +from fate_client.pipeline.adapters.bfia.components.unionpay.hetero_lr import HeteroLR +from fate_client.pipeline.interface import DataWarehouseChannel + + +pipeline = FateFlowPipeline().set_parties( + guest="JG0100001100000010", + host="JG0100001100000010", + arbiter="JG0100001100000010" +) +pipeline.set_site_role("guest") +pipeline.set_site_party_id("JG0100001100000010") + +intersection_0 = Intersection( + "intersect_rsa_1", + id="id", + intersect_method="rsa", + only_output_key=False, + rsa_params=dict( + final_hash_method="sha256", + hash_method="sha256", + key_length=2048 + ), + sync_intersect_ids=True, + connect_engine="mesh", + train_data=[ + DataWarehouseChannel(dataset_id="testspace#test_guest", parties=dict(guest="JG0100001100000010")), + DataWarehouseChannel(dataset_id="testspace#test_host", parties=dict(host="JG0100001100000010")) + ] +) + +hetero_lr_0 = HeteroLR( + "hetero_logistic_regression_1", + id="id", + label="y", + batch_size=-1, + penalty="L2", + early_stop="diff", + tol=0.0001, + max_iter=2, + alpha=0.01, + optimizer="nesterov_momentum_sgd", + init_param={"init_method":"zeros"}, + learning_rate=0.15, + connect_engine="mesh", + train_data=intersection_0.outputs["train_data"] +) + +pipeline.add_task(intersection_0) +pipeline.add_task(hetero_lr_0) +pipeline.conf.set( + "extra", + dict(initiator={'party_id': 'JG0100001100000010', 'role': 'guest'}) +) + +pipeline.protocol_kind = "bfia" +pipeline.guest.conf.set("resources", dict(cpu=-1, disk=-1, memory=-1)) +pipeline.hosts[0].conf.set("resources", dict(cpu=-1, disk=-1, memory=-1)) +pipeline.compile() +pipeline.fit() diff --git a/examples/bfia/unionpay/pipeline/test_unionpay_sbt.py b/examples/bfia/unionpay/pipeline/test_unionpay_sbt.py new file mode 100644 index 000000000..f7c685015 --- /dev/null +++ b/examples/bfia/unionpay/pipeline/test_unionpay_sbt.py @@ -0,0 +1,77 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_client.pipeline import FateFlowPipeline +from fate_client.pipeline.adapters.bfia.components.unionpay.intersection import Intersection +from fate_client.pipeline.adapters.bfia.components.unionpay.hetero_secureboost import HeteroSecureBoost +from fate_client.pipeline.interface import DataWarehouseChannel + + +pipeline = FateFlowPipeline().set_parties( + guest="JG0100001100000010", + host="JG0100001100000010" +) + +pipeline.set_site_role("guest") +pipeline.set_site_party_id("JG0100001100000010") + +intersection_0 = Intersection( + "intersect_rsa_1", + id="id", + intersect_method="rsa", + only_output_key=False, + rsa_params=dict( + final_hash_method="sha256", + hash_method="sha256", + key_length=2048 + ), + sync_intersect_ids=True, + connect_engine="mesh", + train_data=[ + DataWarehouseChannel(dataset_id="testspace#test_guest", parties=dict(guest="JG0100001100000010")), + DataWarehouseChannel(dataset_id="testspace#test_host", parties=dict(host="JG0100001100000010")) + ] +) + +hetero_sbt_0 = HeteroSecureBoost( + "hetero_secureboost_1", + id="id", + label="y", + learning_rate=0.5, + objective_param={"objective": "cross_entropy"}, + num_trees=2, + subsample_feature_rate=1, + n_iter_no_change=True, + tol=0.0001, + predict_param={"threshold": 0.5}, + cv_param={"n_splits": 5, "shuffle": False, "random_seed": 103, "need_cv": False}, + metrics=["auc", "ks"], + early_stopping_rounds="", + tree_param={"max_depth": 5}, + connect_engine="mesh", + train_data=intersection_0.outputs["train_data"] +) + +pipeline.add_task(intersection_0) +pipeline.add_task(hetero_sbt_0) +pipeline.conf.set( + "extra", + dict(initiator={'party_id': 'JG0100001100000010', 'role': 'guest'}) +) + +pipeline.protocol_kind = "bfia" +pipeline.guest.conf.set("resources", dict(cpu=-1, disk=-1, memory=-1)) +pipeline.hosts[0].conf.set("resources", dict(cpu=-1, disk=-1, memory=-1)) +pipeline.compile() +pipeline.fit() diff --git a/examples/bfia/unionpay/register/unionpay_components.json b/examples/bfia/unionpay/register/unionpay_components.json new file mode 100644 index 000000000..30f97c03f --- /dev/null +++ b/examples/bfia/unionpay/register/unionpay_components.json @@ -0,0 +1,670 @@ +{ + "name": "unionpay", + "device": "docker", + "version": "2.0.0", + "metadata": { + "base_url": "", + "image": "unionpay:2.0.0" + }, + "protocol": "bfia", + "components_description": + { + "Intersection": { + "componentName": "Intersection", + "title": "对齐算法", + "provider": "unionpay", + "version": "2.0.0", + "description": "对齐算法", + "roleList": ["guest", "host"], + "desVersion": "1.3.0", + "storageEngine": ["s3","hdfs"], + "inputParam": [ + { + "name": "id", + "title": "id列", + "description": "id字段名", + "type": "string", + "optional": "true", + "defaultValue": "x1", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "intersect_method", + "title": "对齐方式", + "description": "对齐方式", + "type": "string", + "optional": "true", + "defaultValue": "raw", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "sync_intersect_ids", + "title": "同步对齐id", + "description": "同步对齐id", + "type": "boolean", + "optional": "true", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "only_output_key", + "title": "仅输出id", + "description": "仅输出id", + "type": "boolean", + "optional": "true", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "use_hash", + "title": "是否使用哈希", + "description": "是否使用哈希", + "type": "string", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "optional": "true", + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "hash_method", + "title": "哈希方法", + "description": "哈希方法", + "type": "string", + "optional": "true", + "defaultValue": "sha256", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "salt", + "title": "salt", + "description": "salt", + "type": "string", + "optional": "true", + "defaultValue": "12345", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "base64", + "title": "选择base64", + "description": "是否选择base64方式", + "type": "string", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "optional": "true", + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "join_role", + "title": "参与角色", + "description": "参与角色", + "type": "string", + "optional": "true", + "defaultValue": "host", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [{ + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv","yaml"] + }], + "outputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }, + { + "name": "train-intersection", + "description": "对齐数", + "category": "report", + "dataFormat": ["csv"] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] +}, + "HeteroLR": { + "componentName": "HeteroLR", + "title": "纵向逻辑回归算法", + "provider": "unionpay", + "version": "2.0.0", + "description": "纵向逻辑回归算法", + "roleList": ["guest", "host", "arbiter"], + "desVersion": "1.2.0", + "storageEngine": ["s3","hdfs"], + "inputParam": [ + { + "name": "id", + "title": "id列", + "description": "id字段名", + "type": "string", + "optional": "true", + "defaultValue": "x1", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "label", + "title": "标签", + "description": "label字段名", + "type": "string", + "optional": "true", + "defaultValue": "y", + "validator": "regular-正则项", + "dependsOn": [], + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "penalty", + "title": "正则项", + "description": "正则项", + "type": "string", + "bindingData": [ + { + "label": "L1正则", + "value": "L1" + }, + { + "label": "L2正则", + "value": "L2" + } + ], + "optional": "true", + "defaultValue": "L2", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "tol", + "title": "最小损失值", + "description": "最小损失值", + "type": "float", + "optional": "true", + "defaultValue": "0.0001", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "alpha", + "title": "惩罚因子", + "description": "惩罚因子", + "type": "float", + "optional": "true", + "defaultValue": "0.01", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "optimizer", + "title": "优化方法", + "description": "优化方法", + "type": "string", + "bindingData": [ + { + "label": "rmsprop", + "value": "rmsprop" + }, + { + "label": "sgd", + "value": "sgd" + }, + { + "label": "adam", + "value": "adam" + }, + { + "label": "sqn", + "value": "sqn" + }, + { + "label": "adagrad", + "value": "adagrad" + } + ], + "optional": "true", + "defaultValue": "rmsprop", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "batch_size", + "title": "批量梯度下降样本量", + "description": "每轮迭代抽取数据计算梯度的size", + "type": "integer", + "bindingData": [ + { + "label": "all", + "value": "all" + }, + { + "label": "2048", + "value": "2048" + }, + { + "label": "4096", + "value": "4096" + }, + { + "label": "8192", + "value": "8192" + } + ], + "optional": "true", + "defaultValue": "2048", + "validator": "(0,1000)", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "learning_rate", + "title": "学习率", + "description": "学习率", + "type": "float", + "optional": "true", + "defaultValue": "0.15", + "validator": "regular-正则项", + "dependsOn": ["optimizer.sgd", "optimizer.adam"], + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "init_param", + "title": "初始化方式", + "description": "初始化方式", + "type": "string", + "optional": "true", + "defaultValue": "zeros", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "max_iter", + "title": "迭代次数", + "description": "迭代次数", + "type": "integer", + "optional": "true", + "defaultValue": "30", + "validator": "(0,1000)", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "early_stop", + "title": "早停策略", + "description": "早停策略", + "type": "string", + "bindingData": [ + { + "label": "weight_diff", + "value": "weight_diff" + }, + { + "label": "diff", + "value": "diff" + } + ], + "optional": "true", + "defaultValue": "weight_diff", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + } + ], + "inputData": [{ + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }], + "outputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }, + { + "name": "model", + "description": "模型文件", + "category": "model", + "dataFormat": ["csv"] + }, + { + "name": "train-loss", + "description": "loss值", + "category": "metric", + "dataFormat": ["json"] + } + ], + "result": [{ + "resultCode": "4444", + "resultMessage": "算法执行失败" + }] +}, + "HeteroSecureBoost":{ + "componentName": "HeteroSecureBoost", + "title": "XGBoost算法", + "provider": "unionpay", + "version": "2.0.0", + "description": "XGBoost算法", + "roleList": ["guest", "host"], + "desVersion": "1.3.0", + "storageEngine": ["s3","hdfs"], + "inputParam": [ + { + "name": "objective_param", + "title": "目标参数", + "description": "目标参数", + "type": "string", + "optional": "true", + "bindingData": [ + { + "label": "cross_entropy", + "value": "cross_entropy" + } + ], + "defaultValue": "cross_entropy", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "learning_rate", + "title": "学习率", + "description": "学习率", + "type": "float", + "optional": "true", + "defaultValue": "0.15", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "num_trees", + "title": "树个数", + "description": "树个数", + "type": "integer", + "optional": "true", + "defaultValue": "5", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "subsample_feature_rate", + "title": "子样本率", + "description": "子样本率", + "type": "integer", + "optional": "true", + "defaultValue": "1", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "n_iter_no_change", + "title": "n轮无变化", + "description": "n轮无变化", + "type": "boolean", + "optional": "true", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "tol", + "title": "停止容忍度", + "description": "停止容忍度", + "type": "float", + "optional": "true", + "defaultValue": "0.0001", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "bin_num", + "title": "分位数", + "description": "分位数", + "type": "integer", + "optional": "true", + "defaultValue": "32", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "predict_param", + "title": "预测参数", + "description": "预测参数", + "type": "string", + "optional": "true", + "defaultValue": "{\"threshold\": 0.5}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "cv_param", + "title": "cv参数", + "description": "cv参数", + "type": "string", + "optional": "true", + "defaultValue": "{\"n_splits\": 5, \"shuffle\": false, \"random_seed\": 103, \"need_cv\": false}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "metrics", + "title": "计算指标", + "description": "计算指标", + "type": "string", + "optional": "true", + "defaultValue": "[\"auc\", \"ks\"]", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "early_stop", + "title": "早停策略", + "description": "早停策略", + "type": "string", + "optional": "true", + "bindingData": [ + { + "label": "weight_diff", + "value": "weight_diff" + }, + { + "label": "diff", + "value": "diff" + } + ], + "defaultValue": "weight_diff", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "tree_param", + "title": "树参数", + "description": "树参数", + "type": "string", + "optional": "true", + "defaultValue": "{\"max_depth\": 3}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [{ + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }], + "outputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }, + { + "name": "model", + "description": "模型文件", + "category": "model", + "dataFormat": ["csv"] + }, + { + "name": "train-loss", + "description": "loss值", + "category": "report", + "dataFormat": ["json"] + }, + { + "name": "train-auc", + "description": "auc ks值", + "category": "report", + "dataFormat": ["json"] + }, + { + "name": "train-ks", + "description": "ks曲线值", + "category": "report", + "dataFormat": ["json"] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] +} + } + +} \ No newline at end of file diff --git a/examples/connector/create_or_update.json b/examples/connector/create_or_update.json deleted file mode 100644 index 473b68bae..000000000 --- a/examples/connector/create_or_update.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "connector_name": "test", - "engine": "MYSQL", - "connector_info": { - "user": "fate", - "passwd": "fate", - "host": "127.0.0.1", - "port": 3306 - } -} \ No newline at end of file diff --git a/examples/dag/lr/train_lr.yaml b/examples/dag/lr/train_lr.yaml new file mode 100644 index 000000000..604752a94 --- /dev/null +++ b/examples/dag/lr/train_lr.yaml @@ -0,0 +1,204 @@ +dag: + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host + - party_id: + - '9998' + role: arbiter + party_tasks: + guest_9999: + parties: + - party_id: + - '9999' + role: guest + tasks: + reader_0: + parameters: + name: breast_hetero_guest + namespace: experiment + host_9998: + parties: + - party_id: + - '9998' + role: host + tasks: + reader_0: + parameters: + name: breast_hetero_host + namespace: experiment + stage: train + tasks: + reader_0: + component_ref: reader + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host + stage: default + binning_0: + component_ref: hetero_feature_binning + dependent_tasks: + - scale_0 + inputs: + data: + train_data: + task_output_artifact: + output_artifact_key: train_output_data + producer_task: scale_0 + model: {} + parameters: + adjustment_factor: 0.5 + bin_col: null + bin_idx: null + category_col: null + category_idx: null + local_only: false + method: quantile + n_bins: 10 + relative_error: 1.0e-06 + skip_metrics: false + split_pt_dict: null + transform_method: null + use_anonymous: false + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host + evaluation_0: + component_ref: evaluation + dependent_tasks: + - lr_0 + inputs: + data: + input_data: + task_output_artifact: + - output_artifact_key: train_output_data + producer_task: lr_0 + parties: + - party_id: + - '9999' + role: guest + parameters: + default_eval_setting: binary + label_column_name: null + metrics: null + predict_column_name: null + parties: + - party_id: + - '9999' + role: guest + stage: default + lr_0: + component_ref: coordinated_lr + dependent_tasks: + - selection_0 + inputs: + data: + train_data: + task_output_artifact: + output_artifact_key: train_output_data + producer_task: selection_0 + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host + model: {} + parameters: + batch_size: null + early_stop: diff + epochs: 10 + floating_point_precision: 23 + output_cv_data: true + threshold: 0.5 + tol: 0.0001 + psi_0: + component_ref: psi + inputs: + data: + input_data: + task_output_artifact: + output_artifact_key: output_data + producer_task: reader_0 + parameters: {} + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host + stage: default + scale_0: + component_ref: feature_scale + dependent_tasks: + - psi_0 + inputs: + data: + train_data: + task_output_artifact: + output_artifact_key: output_data + producer_task: psi_0 + model: {} + parameters: + feature_range: null + method: standard + scale_col: null + scale_idx: null + strict_range: true + use_anonymous: false + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host + selection_0: + component_ref: hetero_feature_selection + dependent_tasks: + - binning_0 + - scale_0 + inputs: + data: + train_data: + task_output_artifact: + output_artifact_key: train_output_data + producer_task: scale_0 + model: + input_models: + task_output_artifact: + - output_artifact_key: output_model + producer_task: binning_0 + parameters: + iv_param: + filter_type: threshold + metrics: iv + threshold: 0.1 + keep_one: true + manual_param: null + method: + - iv + select_col: null + statistic_param: null + use_anonymous: false + parties: + - party_id: + - '9999' + role: guest + - party_id: + - '9998' + role: host +schema_version: 2.0.0.beta diff --git a/examples/data/README.md b/examples/data/README.md deleted file mode 100644 index 310f70bc0..000000000 --- a/examples/data/README.md +++ /dev/null @@ -1,289 +0,0 @@ -# Data - -This document provides explanation and source information on data used for running examples. -Many of the data sets have been scaled or transformed from their original version. - -## Data Set Naming Rule -All data sets are named according to this guideline: - name: "{content}\_{mode}\_{size}\_{role}\_{role_index}" - -- content: brief description of data content -- mode: how original data is divided, either "homo""or hetero"; some data sets do not have this information -- size: includes keyword "mini" if the data set is truncated from another larger set -- role: role name, either "host" or "guest" -- role_index: if a data set is further divided and shared among multiple hosts in some example, -indices are used to distinguish different parties, starts at 1 - -Data sets used for running examples are uploaded to FATE data storage at the time of deployment. -Uploaded tables share the same `namespace` "experiment" and have `table_name` matching to original file names. -Below lists example data sets and their information. - - -## Horizontally Divided Data -> For Homogeneous Federated Learning - -#### breast_homo: -- 30 features -- label type: binary -- [source](https://www.kaggle.com/uciml/breast-cancer-wisconsin-data) -- data sets: - 1. "breast_homo_guest.csv" - * name: "breast_homo_guest" - * namespace: "experiment" - 2. "breast_homo_host.csv" - * name: "breast_homo_host" - * namespace: "experiment" - 3. "breast_homo_test.csv" - * name: "breast_homo_test" - * namespace: "experiment" - -#### default_credit_homo: -- 23 features -- label type: binary -- [source](https://archive.ics.uci.edu/ml/datasets/default+of+credit+card+clients) -- data sets: - 1. "default_credit_homo_guest.csv" - * name: "default_credit_homo_guest" - * namespace: "experiment" - 2. "default_credit_homo_host_1.csv" - * name: "default_credit_homo_host_1" - * namespace: "experiment" - 3. "default_credit_homo_host_2.csv" - * name: "default_credit_homo_host_2" - * namespace: "experiment" - 4. "default_credit_homo_test.csv" - * name: "defeault_credit_homo_test" - * namespace: "experiment" - -#### epsilon_5k_homo: -- 100 features -- label type: binary -- mock data -- data sets: - 1. "epsilon_5k_homo_guest.csv" - * name: "epsilon_5k_homo_guest" - * namespace: "experiment" - 2. "epsilon_5k_homo_hostt.csv" - * name: "epsilon_5k_homo_host" - * namespace: "experiment" - 3. "epsilon_5k_homo_test.csv" - * name: "epsilon_5k_homo_test" - * namespace: "experiment" - -#### give_credit_homo: -- 10 features -- label type: binary -- [source](https://www.kaggle.com/c/GiveMeSomeCredit/data) -- data sets: - 1. "give_credit_homo_guest.csv" - * name: "give_credit_homo_guest" - * namespace: "experiment" - 2. "give_credit_homo_host.csv" - * name: "give_credit_homo_host" - * namespace: "experiment" - 3. "give_credit_homo_test.csv" - * name: "give_credit_homo_test" - * namespace: "experiment" - -#### student_homo: -- 13 features -- label type: continuous -- [source](https://archive.ics.uci.edu/ml/datasets/student+performance) -- data sets: - 1. "student_homo_guest.csv" - * name: "student_homo_guest" - * namespace: "experiment" - 2. "student_homo_host.csv" - * name: "student_homo_host" - * namespace: "experiment" - 3. "student_homo_test.csv" - * name: "student_homo_test" - * namespace: "experiment" - -#### vehicle\_scale_homo: -- 18 features -- label type: multi-class -- [source](https://archive.ics.uci.edu/ml/datasets/Statlog+(Vehicle+Silhouettes)) -- data sets: - 1. "vehicle_scale_homo_guest.csv" - * name: "vehicle_scale_homo_guest" - * namespace: "experiment" - 2. "vehicle_scale_homo_host.csv" - * name: "vehicle_scale_homo_host" - * namespace: "experiment" - 3. "vehicle_scale_homo_test.csv" - * name: "vehicle_scale_homo_test" - * namespace: "experiment" - -## Vertically Divided Data -> For Heterogeneous Federated Learning - -#### breast_hetero: -- 30 features -- label type: binary -- [source](https://www.kaggle.com/uciml/breast-cancer-wisconsin-data) -- data sets: - 1. "breast_hetero_guest.csv" - * name: "breast_hetero_guest" - * namespace: "experiment" - 2. "breast_hetero_host.csv" - * name: "breast_hetero_host" - * namespace: "experiment" - -#### breast_hetero_mini: -- 7 features -- label type: binary -- [source](https://www.kaggle.com/uciml/breast-cancer-wisconsin-data) -- data sets: - 1. "breast_hetero_mini_guest.csv" - * name: "breast_hetero_mini_guest" - * namespace: "experiment" - 2. "breast_hetero_mini_host.csv" - * name: "breast_hetero_mini_host" - * namespace: "experiment" - -#### default_credit_hetero: -- 23 features -- label type: binary -- [source](https://archive.ics.uci.edu/ml/datasets/default+of+credit+card+clients) -- data sets: - 1. "default_credit_hetero_guest.csv" - * name: "default_credit_hetero_guest" - * namespace: "experiment" - 2. "default_credit_hetero_host.csv" - * name: "default_credit_hetero_host" - * namespace: "experiment" - -#### dvisits_hetero: -- 12 features -- label type: continuous -- [source](https://www.rdocumentation.org/packages/faraway/versions/1.0.7/topics/dvisits) -- data sets: - 1. "dvisits_hetero_guest.csv" - * name: "dvisits_hetero_guest" - * namespace: "experiment" - 2. "dvisits_hetero_host.csv" - * name: "dvisits_hetero_host" - * namespace: "experiment" - -#### epsilon_5k_hetero: -- 100 features -- label type: binary -- mock data -- data sets: - 1. "epsilon_5k_hetero_guest.csv" - * name: "epsilon_5k_hetero_guest" - * namespace: "experiment" - 2. "epsilon_5k_hetero_host.csv" - * name: "epsilon_5k_hetero_host" - * namespace: "experiment" - -#### give_credit_hetero: -- 10 features -- label type: binary -- [source](https://www.kaggle.com/c/GiveMeSomeCredit/data) -- data sets: - 1. "give_credit_hetero_guest.csv" - * name: "give_credit_hetero_guest" - * namespace: "experiment" - 2. "give_credit_hetero_host.csv" - * name: "give_credit_hetero_host" - * namespace: "experiment" - -#### ionosphere_scale_hetero -- 34 features -- label type: binary -- [source](https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/ionosphere_scale) -- data sets: - 1. "ionosphere_scale_hetero_guest.csv" - * name: "ionosphere_scale_hetero_guest" - * namespace: "experiment" - 2. "ionosphere_scale_hetero_host.csv" - * name: "ionosphere_scale_hetero_host" - * namespace: "experiment" - -#### motor_hetero: -- 11 features -- label type: continuous -- [source](https://www.kaggle.com/wkirgsn/electric-motor-temperature) -- data sets: - 1. "motor_hetero_guest.csv" - * name: "motor_hetero_guest" - * namespace: "experiment" - 2. "motor_hetero_host.csv" - * name: "motor_hetero_host" - * namespace: "experiment" - 3. "motor_hetero_host_1.csv" - * name: "motor_hetero_host_1" - * namespace: "experiment" - 4. "motor_hetero_host_2.csv" - * name: "motor_hetero_host_2" - * namespace: "experiment" - -#### motor_hetero_mini: -- 7 features -- label type: continuous -- [source](https://www.kaggle.com/wkirgsn/electric-motor-temperature) -- data sets: - 1. "motor_hetero_mini_guest.csv" - * name: "motor_hetero_mini_guest" - * namespace: "experiment" - 2. "motor_hetero_mini_host.csv" - * name: "motor_hetero_mini_host" - * namespace: "experiment" - -#### student_hetero: -- 13 features -- label type: continuous -- [source](https://archive.ics.uci.edu/ml/datasets/student+performance) -- data sets: - 1. "student_hetero_guest.csv" - * name: "student_hetero_guest" - * namespace: "experiment" - 2. "student_hetero_host.csv" - * name: "student_hetero_host" - * namespace: "experiment" - -#### vehicle_scale_hetero: -- 18 features -- label type: multi-class -- [source](https://archive.ics.uci.edu/ml/datasets/Statlog+(Vehicle+Silhouettes)) -- data sets: - 1. "vehicle_scale_hetero_guest.csv" - * name: "vehicle_scale_hetero_guest" - * namespace: "experiment" - 2. "vehicle_scale_hetero_host.csv" - * name: "vehicle_scale_hetero_host" - * namespace: "experiment" - - -## Federated Transfer Learning Data -> For Federated Transfer Learning - -### nus_wide: -- 636/1000 features -- label type: binary -- [source](https://lms.comp.nus.edu.sg/wp-content/uploads/2019/research/nuswide/NUS-WIDE.html) -- data sets: - 1. "nus_wide_train_guest.csv" - * name: "nus_wide_train_guest" - * namespace: "experiment" - 2. "nus_wide_train_host.csv" - * name: "nus_wide_train_host" - * namespace: "experiment" - 3. "nus_wide_validate_guest.csv" - * name: "nus_wide_validate_guest" - * namespace: "experiment" - 4. "nus_wide_validate_guest.csv" - * name: "nus_wide_validate_guest" - * namespace: "experiment" - - -## Non-Divided Data -> Generated Data for Data Operation Demo - -#### tag_value: -- data sets: - 1. "tag_value_1000_140.csv" - * name: "tag_value_1", "tag_value_2", "tag_value_3" - * namespace: "experiment" diff --git a/examples/data/breast_hetero_mini_guest.csv b/examples/data/breast_hetero_mini_guest.csv deleted file mode 100644 index 51eda0bc4..000000000 --- a/examples/data/breast_hetero_mini_guest.csv +++ /dev/null @@ -1,570 +0,0 @@ -id,y,x0,x1,x2 -133,1,0.254879,-1.046633,0.209656 -273,1,-1.142928,-0.781198,-1.166747 -175,1,-1.451067,-1.406518,-1.456564 -551,1,-0.879933,0.420589,-0.877527 -199,0,0.426758,0.723479,0.316885 -274,0,0.963102,1.467675,0.829202 -420,1,-0.662496,0.212149,-0.620475 -76,1,-0.453343,-2.147457,-0.473631 -315,1,-0.606584,-0.971725,-0.678558 -399,1,-0.583805,-0.193332,-0.633283 -238,1,-0.107515,2.420311,-0.141817 -246,1,-0.482335,0.348938,-0.565371 -253,0,0.741523,-0.095626,0.704101 -550,1,-0.954483,-0.147736,-0.98833 -208,1,-0.356014,0.567149,-0.23177 -185,1,-0.910995,-0.732345,-0.949311 -156,0,0.869914,-0.092369,0.763673 -0,0,1.88669,-1.359293,2.303601 -70,0,1.779007,0.147012,1.746605 -293,1,-0.664567,0.011851,-0.68243 -287,1,-0.548601,-1.650784,-0.591583 -222,1,-1.055953,-0.462024,-1.052072 -262,0,0.853348,0.254488,0.912602 -309,1,-0.318739,-1.347894,-0.396188 -534,1,-0.962766,0.135613,-0.918334 -54,0,0.379129,0.979143,0.310928 -172,0,0.522016,-1.406518,0.528365 -484,1,0.153409,-1.868995,0.156042 -102,1,-0.606584,1.166414,-0.675579 -458,1,-0.399502,1.010084,-0.482567 -495,1,-0.053674,0.456415,-0.100117 -158,1,-0.648001,-1.183422,-0.690472 -160,1,-0.610726,0.086759,-0.546606 -292,1,-0.523751,-0.9359,-0.549585 -493,1,-0.637646,-1.517252,-0.715492 -394,1,-0.561026,0.019993,-0.563882 -528,1,-0.341518,-1.676839,-0.379508 -335,0,0.977597,1.216895,1.070467 -311,1,0.039513,-0.639524,-0.106074 -25,0,1.238521,-0.696519,1.344497 -522,1,-0.89857,0.122585,-0.919823 -512,0,0.029159,0.64857,0.17987 -230,0,0.687682,-0.128194,0.781544 -548,1,-1.105653,-0.014204,-1.136664 -213,0,0.372916,0.389649,0.39135 -390,1,-1.012466,-1.632871,-1.013648 -275,1,-0.801242,-1.088973,-0.828082 -219,0,2.408538,3.21336,2.172543 -236,0,3.052564,1.438363,2.941018 -336,1,-0.527893,-1.427688,-0.592179 -20,1,-0.366368,-0.844707,-0.332744 -193,0,-0.128223,2.224898,-0.165645 -200,1,-0.378793,0.436874,-0.4501 -83,0,0.840923,1.146872,1.013874 -376,1,-1.12222,-0.465281,-0.915951 -295,1,-0.331164,-1.424431,-0.389933 -77,0,1.267513,-1.102,1.275989 -441,0,0.851277,1.593064,0.760694 -109,1,-0.674921,0.56552,-0.693153 -319,1,-0.6977,-0.890303,-0.759575 -349,1,-0.716338,-1.295784,-0.71996 -89,1,0.014664,-1.211106,0.063706 -419,1,-0.809525,0.528066,-0.83404 -562,0,0.259021,2.786709,0.638572 -370,0,0.644194,0.871666,0.656444 -546,1,-1.039387,-0.636267,-1.076496 -49,1,-0.231765,1.000313,-0.246067 -412,1,-1.305488,0.376621,-1.21083 -417,0,1.429037,0.321254,1.48449 -440,1,-0.809525,0.194236,-0.50997 -482,1,-0.298031,-1.198078,-0.366998 -291,1,-0.003974,0.083503,0.05477 -198,0,1.468383,1.039396,1.761498 -542,1,0.049868,1.07685,0.004134 -320,1,-1.033174,-0.825166,-1.064284 -94,0,0.40605,-0.235671,0.483686 -116,1,-1.41959,-1.401633,-1.30823 -429,1,-0.507184,-0.768171,-0.547798 -518,1,-0.252473,-0.212873,-0.236834 -323,0,1.870123,1.006827,1.901492 -248,1,-0.832304,1.549097,-0.872165 -280,0,1.542933,1.664716,1.564912 -391,1,-1.263036,-0.468538,-1.288274 -462,1,-0.179994,1.026368,-0.204367 -406,1,0.298367,-0.992895,0.257314 -416,1,-1.12222,0.905864,-1.147684 -411,1,-0.799171,0.124213,-0.814083 -474,1,-0.8965,-1.030349,-0.788765 -96,1,-0.712196,-0.774684,-0.748256 -232,1,-0.809525,2.622237,-0.858464 -354,1,-0.859225,-1.605188,-0.823317 -240,1,-0.293889,-1.079202,-0.39172 -378,1,-0.358085,-0.983124,-0.277044 -177,0,0.314933,0.451529,0.483686 -33,0,1.631978,0.850497,1.612569 -251,1,-0.683205,-0.523905,-0.719066 -224,1,-0.233835,-0.338263,-0.250833 -186,0,1.043864,0.111186,0.951324 -85,0,1.379337,0.32614,1.338539 -422,1,-0.751542,-0.978239,-0.754511 -104,1,-0.979333,-0.385488,-0.98416 -461,0,4.094189,0.927033,4.287337 -142,1,-0.72255,0.176323,-0.732768 -380,1,-0.710125,-0.838193,-0.665154 -123,1,-0.117869,-1.579133,-0.132881 -428,1,-0.950341,-0.877276,-0.980288 -396,1,-0.304244,0.247975,-0.295809 -66,1,-1.213336,0.957974,-1.19832 -170,1,-0.573451,-1.634499,-0.604391 -481,1,0.029159,0.120957,-0.085224 -313,1,-0.813667,-2.085577,-0.775361 -53,0,0.896835,-0.251956,0.829202 -296,1,-1.014537,-1.768031,-1.037775 -114,1,-1.375274,-0.986381,-1.274274 -243,1,-0.260756,0.107929,-0.275853 -140,1,-1.169849,-1.885279,-1.213213 -152,1,-1.087016,-1.007551,-1.078879 -268,1,-0.490618,-0.331749,-0.535883 -91,0,0.033301,0.026507,0.007112 -255,0,0.025018,-0.587414,0.024984 -559,1,-0.784675,1.869899,-0.744086 -14,0,-0.256615,1.031253,0.045834 -100,0,0.149267,1.562124,0.039877 -353,0,0.464033,1.228294,0.415178 -543,1,-0.393289,1.871527,-0.440271 -165,1,-0.059886,0.02325,-0.147774 -43,0,0.230029,0.37825,0.173913 -201,0,0.85956,0.026507,0.960259 -108,0,2.512079,0.379878,2.964846 -328,0,0.623486,0.765818,0.671337 -439,1,-0.281464,-1.036863,-0.319638 -162,0,2.166251,0.116071,2.014678 -325,1,-0.529963,-0.745372,-0.552861 -183,1,-0.807454,-1.299041,-0.83821 -197,0,0.722886,-0.159135,0.650487 -498,0,1.342063,-0.45551,1.165782 -477,1,-0.233835,-0.631382,-0.180538 -421,1,0.039513,-1.194821,0.203699 -374,1,-0.29596,-0.890303,-0.241301 -470,1,-1.062166,-0.009318,-1.083645 -382,1,-0.766038,0.493869,-0.592774 -510,1,-0.790887,-1.315326,-0.774766 -132,0,0.662832,0.977515,0.668358 -545,1,-0.190348,0.55575,-0.288363 -121,0,1.238521,-0.126566,1.135996 -45,0,1.356558,-0.709547,1.290882 -333,1,-0.726692,-0.589042,-0.750044 -134,0,1.294434,0.93029,1.141953 -182,0,0.795365,1.163157,0.656444 -519,1,-0.376722,-0.641152,-0.406017 -207,0,0.731169,-0.102139,0.677294 -99,0,0.012593,0.843983,0.066684 -366,0,1.640261,1.324372,1.570869 -180,0,3.489508,1.168042,3.381848 -318,1,-1.285815,-0.370832,-1.150961 -501,0,-0.053674,1.182698,-0.037566 -229,0,-0.221411,0.728364,-0.058416 -7,0,0.163763,0.401048,0.099449 -247,1,-0.389147,-1.299041,-0.067352 -494,1,-0.366368,0.453158,-0.356573 -174,1,-0.979333,-1.054776,-1.014542 -340,1,0.083001,-0.678606,0.123277 -125,1,-0.161357,-0.34152,-0.207346 -209,1,0.230029,-1.588903,0.191785 -347,1,0.20725,-1.261587,0.206678 -332,1,-0.888216,0.016737,-0.904036 -361,1,-0.428493,0.573662,-0.426569 -500,1,0.101638,-0.854478,0.072641 -300,0,2.000585,0.091645,1.901492 -283,0,0.472316,-0.095626,0.584958 -107,1,-0.616938,0.295199,-0.646389 -131,0,0.619345,0.052562,0.525386 -303,1,-1.078732,-0.18519,-1.087219 -567,0,1.961239,2.237926,2.303601 -507,1,-0.94827,-0.803996,-0.928759 -3,0,-0.281464,0.133984,-0.249939 -516,0,1.157759,0.085131,1.040681 -143,1,-0.37051,-0.628125,-0.300575 -398,1,-0.743259,-0.867505,-0.788467 -530,1,-0.573451,0.374993,-0.558223 -119,0,0.892693,0.350566,0.653465 -259,0,0.459891,3.885905,0.567086 -450,1,-0.720479,0.407562,-0.70745 -192,1,-1.304866,-0.78934,-1.340697 -514,0,0.271446,0.38802,0.194763 -241,1,-0.635576,-0.864248,-0.697323 -402,1,-0.442989,-0.173791,-0.326191 -148,1,-0.086807,-0.948927,0.039877 -395,1,-0.279394,-0.054915,-0.322915 -299,1,-1.105653,-0.2373,-1.106878 -149,1,-0.192419,-0.523905,-0.29998 -60,1,-1.087016,-1.339752,-1.114026 -61,1,-1.388321,0.22192,-1.346356 -126,0,0.128559,1.622376,0.176892 -372,0,1.329638,-0.624868,1.335561 -82,0,2.843411,1.293432,3.110797 -425,1,-1.068378,0.531323,-1.112239 -337,0,1.71274,1.415565,1.603633 -386,1,-0.650071,-1.04012,-0.584136 -184,0,0.317004,0.383135,0.194763 -58,1,-0.422281,-0.558102,-0.506991 -88,1,-0.505114,0.785359,-0.470652 -117,0,0.526157,0.275658,0.590915 -459,1,-1.159494,1.830816,-1.168535 -221,1,-0.266969,-1.391862,-0.183517 -223,0,0.681469,0.751162,0.555172 -128,1,-0.032965,-1.19645,-0.040545 -294,1,-0.573451,-1.334867,-0.557627 -541,1,-0.010186,0.985657,0.185828 -111,1,-0.608655,-0.033745,-0.543926 -266,1,-0.908925,-0.44574,-0.86323 -568,1,-1.410893,0.76419,-1.432735 -362,1,-0.52168,0.050934,-0.579073 -271,1,-0.817808,-1.546564,-0.863528 -187,1,-0.674921,-0.698148,-0.680345 -155,1,-0.554813,-0.074456,-0.615412 -5,0,-0.165498,-0.313836,-0.115009 -511,1,-0.136507,-1.318583,-0.165645 -252,0,1.865981,-0.014204,1.564912 -480,1,-0.606584,0.35708,-0.548989 -513,1,0.101638,-1.373949,0.036898 -86,0,-0.012257,0.581805,0.03392 -105,0,0.008451,-0.533675,-0.025652 -242,1,-0.763967,0.371736,-0.598731 -312,1,-0.430564,-1.510738,-0.453377 -473,1,-0.583805,2.01483,-0.660686 -364,1,-0.318739,-0.647666,-0.402145 -565,0,1.53672,2.047399,1.42194 -447,1,0.033301,-0.478309,-0.040545 -488,1,-0.610726,-0.665579,-0.616305 -151,1,-1.486271,0.658341,-1.464904 -263,0,0.339783,0.975886,0.257314 -563,0,1.66097,0.60786,2.139779 -194,0,-0.039178,0.342424,0.337735 -433,0,1.323425,0.855382,1.133017 -244,0,1.114272,0.790245,1.121103 -489,0,0.602778,0.143755,0.596872 -465,1,-0.171711,-0.02886,0.230506 -483,1,-0.27111,-0.349662,-0.341978 -397,1,-0.523751,-0.751886,-0.492694 -52,1,-0.656284,-0.707918,-0.702684 -147,1,-0.003974,-0.033745,-0.004802 -27,0,1.043864,0.257745,0.972174 -436,1,-0.376722,-0.211245,-0.36104 -434,1,0.008451,-0.836565,-0.147774 -190,0,-0.109586,1.873156,-0.025652 -407,1,-0.387077,0.217034,-0.465589 -202,0,1.832848,1.140359,2.077228 -455,1,-0.252473,2.594554,-0.314872 -452,1,-0.658355,1.987146,-0.660984 -385,0,-0.099232,0.9824,-0.150752 -188,1,-0.766038,0.130727,-0.824806 -159,1,-0.809525,-1.217619,-0.869485 -32,0,0.954818,1.044281,0.858987 -264,0,1.099776,0.594832,0.990045 -245,1,-0.991758,0.616002,-1.000245 -535,0,1.66304,-0.032117,1.576826 -430,0,0.016734,0.308227,0.540279 -443,1,-1.103582,-0.385488,-1.129217 -154,1,-0.310456,-0.843079,-0.285682 -9,0,-0.24419,2.443109,-0.286278 -63,1,-1.296169,-1.04989,-1.241212 -329,0,0.302508,-0.076084,0.191785 -536,0,-0.202773,1.39928,-0.088202 -65,0,0.215534,1.255978,0.218592 -438,1,-0.132365,0.379878,-0.189474 -269,1,-0.94827,-0.076084,-0.915951 -36,0,-0.078524,0.762561,0.266249 -98,1,-0.664567,-1.386977,-0.723832 -392,0,1.021085,0.60786,1.037702 -69,1,-0.581734,-0.963583,-0.643112 -517,0,1.545003,-0.072828,1.585762 -92,1,0.018805,-0.541818,-0.082245 -552,1,-0.49683,1.681,-0.570733 -505,1,-1.17399,-1.243674,-1.125643 -566,0,0.561361,1.374854,0.579001 -163,1,-0.556884,0.488984,-0.592774 -358,1,-1.302174,-1.299041,-1.250743 -442,1,-0.206915,-1.33161,-0.278832 -15,0,0.246596,1.865014,0.501557 -503,0,3.007006,-0.294295,3.10484 -176,1,-1.037316,-0.209616,-1.018414 -327,1,-0.662496,-0.558102,-0.730385 -1,0,1.805927,-0.369203,1.535126 -404,1,-0.639717,-1.437458,-0.689578 -24,0,2.110339,0.957974,2.077228 -139,1,-0.900641,-1.61333,-0.915355 -101,1,-1.726901,-0.999409,-1.693361 -350,1,-0.619009,-0.96684,-0.704471 -90,1,-0.032965,0.559006,-0.129902 -191,1,-0.52168,-0.354547,-0.542734 -383,1,-0.432635,-0.414799,-0.35836 -11,0,0.85956,0.261002,0.870902 -314,1,-1.515262,-0.527162,-1.507497 -352,0,3.491579,-0.34152,3.635028 -235,1,-0.19449,0.749534,-0.267811 -305,1,-0.792958,0.967744,-0.770596 -375,1,0.145126,-1.064546,0.173913 -215,0,-0.107515,0.204007,-0.085224 -466,1,-0.304244,-0.035373,-0.189474 -486,1,0.039513,-0.03863,-0.037566 -250,0,1.928106,0.215406,1.728734 -339,0,2.982156,0.822813,2.833789 -103,1,-1.140857,0.187723,-1.043732 -206,1,-1.211265,-0.400144,-1.196831 -345,1,-1.116007,-1.009179,-1.083347 -346,1,-0.544459,0.225177,-0.617199 -317,0,1.153617,-0.110282,1.001959 -499,0,1.571924,0.827699,1.666184 -478,1,-0.801242,-0.615097,-0.751235 -47,0,-0.124082,0.370108,-0.132881 -456,1,-0.652142,2.138591,-0.632092 -211,1,-0.614867,-0.11191,-0.656516 -490,1,-0.434706,1.027997,-0.432526 -113,1,-1.058024,-0.47668,-1.031818 -359,1,-0.879933,-0.107025,-0.937396 -344,1,-0.664567,-1.224133,-0.688089 -532,1,-0.086807,-0.891932,-0.168624 -261,0,0.741523,0.943318,0.623679 -4,0,1.298575,-1.46677,1.338539 -226,1,-0.983474,-0.957069,-1.0065 -464,1,-0.283535,-0.291038,-0.362232 -277,0,0.764302,-0.224272,0.647508 -257,0,0.302508,-0.491336,0.373478 -260,0,1.669253,2.195586,1.639376 -301,1,-0.581734,-0.42457,-0.569839 -537,1,-0.681134,1.060565,-0.629709 -403,1,-0.498901,-0.432712,-0.523373 -233,0,1.698245,1.905725,1.651291 -487,0,1.592632,0.767446,1.389175 -141,0,0.756019,-0.066314,0.647508 -68,1,-1.234044,-0.492965,-1.243893 -363,1,0.385341,-0.037002,0.296035 -379,0,-0.627292,1.163157,-0.461717 -51,1,-0.331164,-0.405029,-0.333042 -195,1,-0.494759,-0.598813,-0.490013 -239,0,1.292363,3.125425,1.010895 -161,0,1.192963,-1.281128,1.171739 -169,1,-0.032965,-0.435969,-0.079266 -212,0,2.452025,-1.173652,2.419765 -120,1,-0.714267,-1.580761,-0.700599 -367,1,-0.409856,-0.266612,-0.399464 -171,0,0.354279,0.682768,0.278164 -348,1,-0.778463,-0.795854,-0.821827 -409,1,-0.449201,0.521552,-0.543926 -112,1,-0.200702,-0.317093,-0.00778 -524,1,-1.041457,-0.437598,-0.981182 -62,0,0.290083,0.624144,0.352628 -393,0,2.06271,0.498754,1.928299 -75,0,0.724957,-0.181933,0.641551 -124,1,-0.416068,-0.47668,-0.454866 -231,1,-0.867508,1.314602,-0.81736 -285,1,-0.573451,-0.422942,-0.646389 -448,1,0.00638,0.441759,0.024984 -460,0,1.38555,1.435106,1.335561 -218,0,1.959169,0.48247,1.877663 -467,1,-1.060095,-0.172162,-1.076794 -525,1,-1.407372,-1.176908,-1.309422 -80,1,-0.654213,1.05568,-0.677068 -472,1,0.188613,-1.214362,0.141149 -549,1,-0.67078,0.940061,-0.695833 -527,1,-0.550672,-1.043377,-0.596944 -168,0,1.422825,1.083363,1.430876 -351,0,0.225888,-0.245442,0.361564 -57,0,0.3315,0.817928,0.251356 -560,1,-0.200702,1.220152,-0.210324 -424,1,-1.04767,-0.408286,-1.05654 -355,1,-0.600372,-0.52879,-0.54333 -42,0,1.619553,1.220152,2.089143 -217,1,-0.991758,-0.196589,-0.949013 -286,1,-0.627292,0.262631,-0.448611 -135,0,-0.368439,1.252721,-0.453377 -521,0,2.826844,0.204007,2.932082 -278,1,-0.159286,0.068847,-0.248748 -504,1,-1.240257,-1.513995,-1.138153 -97,1,-1.107724,0.099787,-1.145302 -84,1,-0.538247,-0.126566,-0.580264 -553,1,-1.330337,-0.102139,-1.322527 -469,1,-0.602442,-0.045144,-0.569541 -87,0,1.716882,0.770703,1.35939 -22,0,0.372916,-1.074317,0.531343 -19,1,-0.240048,-1.045005,-0.225217 -265,0,3.359046,3.498337,3.179304 -38,0,-0.264898,-0.077713,-0.349126 -479,0,0.2321,-0.427827,0.441986 -17,0,0.971385,0.944946,0.879838 -189,1,-0.604513,-0.991267,-0.613922 -178,1,-0.46991,0.54435,-0.56835 -164,0,2.431317,0.414075,2.291686 -451,0,1.070784,0.860267,0.969195 -290,1,-0.103373,-0.577643,-0.165645 -67,1,-0.815737,-0.29918,-0.87157 -110,1,-1.080803,-0.68512,-1.059816 -556,1,-1.163636,-0.45551,-1.173002 -150,1,-0.436776,-0.255213,-0.489715 -321,0,1.406258,-0.431084,1.278968 -526,1,-0.190348,-0.084227,-0.159688 -18,0,2.28843,0.84724,2.369129 -446,0,1.089422,2.094623,1.135996 -432,0,1.192963,-0.098883,1.153867 -523,1,-0.240048,-0.00769,-0.233259 -418,1,-0.542388,-1.426059,-0.570137 -272,0,2.468592,0.407562,2.640181 -153,1,-0.886145,-1.527023,-0.923695 -509,0,0.174117,1.734739,0.310928 -288,1,-0.913066,-0.545075,-0.863528 -258,0,0.741523,0.971001,1.08536 -228,1,-0.428493,0.917263,-0.494183 -476,1,0.037443,0.257745,0.144127 -423,1,-0.233835,-0.02886,-0.174581 -64,0,0.169975,1.269005,0.135192 -137,1,-0.817808,-0.595556,-0.814083 -343,0,1.342063,1.462789,1.499383 -302,0,1.534649,0.611116,1.535126 -561,1,-0.900641,2.055541,-0.955268 -8,0,-0.161357,0.822813,-0.031609 -502,1,-0.558955,-0.696519,-0.613327 -28,0,0.828498,1.796619,1.252161 -457,1,-0.397431,1.392767,-0.475716 -136,1,-0.608655,-0.032117,-0.628517 -365,0,1.665111,0.112814,1.606612 -384,1,-0.42021,-1.35278,-0.317851 -81,1,-0.153073,-0.405029,-0.315766 -306,1,-0.385006,-0.851221,-0.454568 -282,0,1.557428,0.484098,1.344497 -475,1,-0.451272,-1.030349,-0.418229 -118,0,0.811931,0.785359,0.68623 -55,1,-0.710125,-0.522276,-0.758086 -557,1,-1.196769,1.394395,-1.214107 -284,1,-0.490618,-0.974982,-0.450994 -256,0,1.818352,1.724968,2.124886 -426,1,-0.857154,-0.668836,-0.77 -387,1,-0.157215,-0.929386,-0.226408 -497,1,-0.457485,-0.217758,-0.430144 -298,1,-0.010186,-0.067942,-0.043523 -157,1,0.403979,0.389649,0.388371 -249,1,-0.749471,-0.730716,-0.785787 -342,1,-0.900641,-0.940785,-0.819147 -508,1,0.217604,-1.289271,0.07562 -267,1,-0.304244,0.710451,-0.28598 -338,1,-1.058024,0.189351,-1.05088 -360,1,-0.527893,-0.764914,-0.608859 -12,0,0.971385,0.694167,1.323647 -210,0,1.443533,0.352195,1.520233 -533,0,1.441462,0.239833,1.332582 -167,0,0.78294,0.101415,0.698144 -79,1,-0.42021,-0.139593,-0.458142 -369,0,2.358838,0.019993,2.613373 -437,1,-0.126153,-0.667207,-0.180538 -485,1,-0.515468,-0.756771,-0.281214 -408,0,0.996235,-0.043516,0.918559 -308,1,-0.26904,-1.422803,-0.350913 -414,0,0.205179,1.829188,0.084556 -41,0,-0.710125,1.573523,-0.596944 -276,1,-0.842658,-1.088973,-0.890335 -146,0,-0.523751,0.114443,-0.456653 -405,1,-0.801242,-0.015832,-0.729789 -35,0,0.774656,0.54435,0.781544 -13,0,0.118205,0.322883,0.141149 -166,1,-0.966908,-2.223994,-1.00084 -173,1,-1.018678,-1.442344,-1.049987 -531,1,-0.604513,0.510153,-0.603497 -388,1,-0.875791,-1.098743,-0.82004 -205,0,0.310792,-0.885418,0.310928 -6,0,1.368983,0.322883,1.368325 -144,1,-0.894429,-0.807253,-0.877825 -540,1,-0.830233,-0.976611,-0.848337 -30,0,1.424896,1.356941,1.585762 -435,0,0.159621,0.834212,0.197742 -31,0,0.114063,0.397791,0.361564 -203,0,0.60692,2.633636,0.632615 -59,1,-1.400331,-1.673582,-1.410693 -95,0,1.646474,0.962859,1.454704 -37,1,-0.614867,-0.466909,-0.679153 -331,1,-0.382935,-0.606955,-0.239812 -237,0,1.646474,0.080246,1.621505 -10,0,0.604849,1.335771,0.492622 -389,0,0.942393,0.775589,1.034724 -324,1,-0.52168,-0.699776,-0.481077 -179,1,-0.54653,-1.551449,-0.612433 -220,1,-0.192419,-1.51888,-0.224919 -50,1,-0.681134,0.006966,-0.723236 -401,1,-0.511326,-0.901702,-0.584434 -368,0,2.998723,0.124213,2.74741 -29,0,0.774656,-1.002666,0.823244 -78,0,1.470454,0.984029,1.877663 -204,1,-0.26904,-0.168905,-0.333935 -214,0,0.122346,1.49373,0.230506 -127,0,1.253017,0.008594,1.219396 -46,1,-1.512777,-0.605327,-1.489328 -554,1,-0.492689,1.638661,-0.548691 -449,0,1.948814,1.041024,1.815113 -564,0,1.901185,0.1177,1.752563 -297,0,-0.602442,-0.37246,-0.66009 -491,1,0.735311,-1.181794,0.590915 -492,0,1.089422,0.062333,1.076424 -181,0,2.155897,1.270634,2.062335 -427,1,-0.726692,1.036139,-0.702088 -471,1,-0.552743,1.246207,-0.596349 -515,1,-0.786746,-0.431084,-0.837316 -445,1,-0.681134,0.762561,-0.678558 -506,1,-0.643859,-0.245442,-0.659197 -93,1,-0.242119,0.042792,-0.288065 -453,1,-0.097161,-1.424431,-0.123945 -357,1,-0.240048,-0.015832,-0.313383 -544,1,-0.252473,-0.150993,-0.241004 -26,0,0.279729,1.226666,0.450921 -356,1,-0.430564,-0.134708,-0.388443 -227,1,0.029159,-1.036863,0.206678 -377,1,-0.327023,1.620748,-0.302362 -234,1,-1.192628,-1.061289,-1.236744 -341,1,-1.142928,-0.42457,-1.072624 -129,0,1.317213,1.286918,1.234289 -289,1,-0.809525,0.07536,-0.833146 -539,1,-1.572003,1.011712,-1.571835 -145,1,-0.64593,-1.492825,-0.625539 -138,0,0.472316,-0.691634,0.421136 -410,1,-0.666638,1.73311,-0.660984 -56,0,2.044072,0.401048,1.871706 -216,1,-0.625221,0.23169,-0.627326 -281,1,-0.612797,-1.207849,-0.672005 -16,0,0.579999,0.84724,0.480707 -444,0,0.851277,-0.595556,0.775587 -547,1,-1.126361,-0.592299,-1.077688 -270,1,-0.281464,-0.818652,-0.381891 -44,0,-0.008116,0.686025,-0.052459 -34,0,0.816073,0.257745,0.757716 -371,1,-0.014328,-1.619844,-0.082245 -310,1,-0.757754,0.142126,-0.784595 -538,1,-1.489377,0.853754,-1.492009 -72,0,1.4601,1.326001,1.320668 -73,0,0.062293,-0.784455,0.090513 -279,1,-0.266969,-0.641152,-0.264832 -196,0,0.025018,1.356941,0.129234 -74,1,-0.44713,-0.401772,-0.522778 -23,0,2.671532,1.614234,2.404872 -334,1,-0.604513,0.453158,-0.677068 -400,0,0.938252,0.342424,1.261096 -431,1,-0.701842,-0.450625,-0.525756 -254,0,1.952956,-0.180304,1.663205 -48,1,-0.519609,-0.81051,-0.517714 -130,1,-0.606584,-1.281128,-0.473035 -555,1,-1.12429,1.5035,-1.122664 -520,1,-1.180203,-1.276243,-1.174194 -122,0,2.019222,-0.274754,2.193393 -496,1,-0.391218,-0.574386,-0.356573 -225,1,0.103709,-1.429316,0.093491 -326,1,-0.153073,-1.250188,-0.263939 -106,1,-0.648001,0.583433,-0.647878 -381,1,-0.865437,-0.78934,-0.82004 -415,1,-0.666638,0.249603,-0.660388 -71,1,-1.353531,-1.629614,-1.331463 -558,1,-0.163427,0.259374,-0.040545 -21,1,-1.250611,-1.631243,-1.254913 -316,1,-0.708054,-1.499339,-0.764341 -463,1,-0.724621,-0.269869,-0.732172 -454,1,-0.399502,-0.574386,-0.465887 -413,1,0.101638,0.956345,0.087534 -330,0,0.515803,-0.60207,0.507515 -468,0,1.097705,0.519924,1.082381 -322,1,-0.461626,-0.748629,-0.430739 -307,1,-1.360572,-0.913101,-1.380908 -373,0,1.884619,-0.408286,1.773413 -304,1,-0.743259,-0.662322,-0.731874 -529,1,-0.583805,-1.61333,-0.60588 -40,0,-0.07024,0.744648,-0.141817 -115,1,-0.538247,0.076989,-0.587413 -2,0,1.51187,-0.023974,1.347475 -39,0,-0.153073,0.055819,0.001155 diff --git a/examples/data/breast_hetero_mini_host.csv b/examples/data/breast_hetero_mini_host.csv deleted file mode 100644 index 66f5a3153..000000000 --- a/examples/data/breast_hetero_mini_host.csv +++ /dev/null @@ -1,570 +0,0 @@ -id,x3,x4,x5 -133,0.074214,-0.441366,-0.377645 -273,-0.923578,0.62823,-1.021418 -175,-1.092337,-0.708765,-1.168557 -551,-0.780484,-1.037534,-0.48388 -199,0.287273,1.000835,0.962702 -274,0.772457,-0.038076,-0.468613 -420,-0.632995,-0.327392,-0.385278 -76,-0.483572,0.558093,-0.740244 -315,-0.591332,-0.963013,-1.302401 -399,-0.560041,-0.34931,-0.519504 -238,-0.204943,-1.063835,-0.074206 -246,-0.489725,-0.976164,-0.658182 -253,0.600181,0.404667,-0.087565 -550,-0.823201,-1.414523,-1.150045 -208,-0.424155,0.110966,1.182806 -185,-0.77978,0.864944,-0.969255 -156,0.740814,0.413434,0.607736 -0,2.001237,1.307686,2.616665 -70,1.732277,-0.572873,-0.131459 -293,-0.637741,0.198638,-0.499147 -287,-0.533673,-1.587236,-0.887829 -222,-0.887716,0.360831,-0.70144 -262,0.728509,-0.831505,0.206332 -309,-0.365968,-1.348769,-1.24553 -534,-0.831639,0.45727,-0.02077 -54,0.262662,0.28631,-0.308942 -172,0.389232,0.90878,0.661808 -484,-0.046203,0.952616,0.277579 -102,-0.585004,-0.879725,-1.053734 -458,-0.44314,-0.463284,-0.92218 -495,-0.170488,-0.472051,-0.734519 -158,-0.611372,-0.213419,-0.833757 -160,-0.59186,0.150419,-0.413905 -292,-0.518906,0.698367,-0.301944 -493,-0.609263,-1.664826,-1.205453 -394,-0.564436,0.474804,-0.489605 -528,-0.399544,0.308228,-0.749786 -335,0.846289,0.549325,-0.311486 -311,-0.069935,-1.370687,-1.166649 -25,1.020322,0.97015,0.894635 -522,-0.781714,-0.945479,-1.12619 -512,-0.063607,1.097274,0.835474 -230,0.54217,1.662757,0.885093 -548,-0.907756,-0.546572,-1.010222 -213,0.246841,-0.353694,-0.476882 -390,-0.854492,0.084665,-0.56785 -275,-0.71755,0.154802,-1.085159 -219,2.806362,0.369598,0.988783 -236,3.627307,0.6896,1.007232 -336,-0.535431,-0.796437,-0.361105 -20,-0.439624,-0.051226,0.148443 -193,-0.196329,2.022211,1.376193 -200,-0.425737,0.461654,-0.318484 -83,0.733782,0.299461,0.174525 -376,-0.929379,-0.792053,0.684709 -295,-0.385832,-0.673696,-0.935539 -77,1.282251,0.676449,1.96653 -441,0.709172,0.492339,1.004687 -109,-0.637214,1.645223,-0.220518 -319,-0.641081,-2.116335,-1.317732 -349,-0.675712,-0.134515,-0.418358 -89,-0.13533,-0.204652,0.347555 -419,-0.742864,-0.182734,-0.912638 -562,0.060502,0.40905,3.418837 -370,0.49998,0.400283,1.350111 -546,-0.871368,-0.169583,-1.055006 -49,-0.319559,-0.708765,-0.529046 -412,-1.018857,-1.041918,-0.417085 -417,1.524843,0.847409,0.92835 -440,-0.710519,0.295077,0.979241 -482,-0.387414,0.303844,-0.027768 -291,-0.124431,-0.046843,0.310022 -198,1.419368,-0.00739,1.945538 -542,-0.095249,-1.155891,-0.742153 -320,-0.861699,0.343297,-0.116191 -94,0.253872,0.996451,1.056214 -116,-1.073352,-0.634244,-0.422174 -429,-0.516445,-1.120822,-1.006469 -518,-0.361925,0.58001,0.266129 -323,1.858847,1.176179,1.240059 -248,-0.746907,0.768505,-0.728158 -280,1.482653,2.009061,0.825932 -391,-0.99073,0.597545,-0.784138 -462,-0.256626,-1.344385,-0.688717 -406,0.118337,-0.515887,-0.522048 -416,-0.916194,0.886862,-0.858566 -411,-0.719308,0.198638,-0.674722 -474,-0.786636,0.036445,0.862192 -96,-0.67747,-0.805204,-1.022181 -232,-0.720187,-1.421536,-1.179499 -354,-0.750775,-1.916882,-0.818489 -240,-0.346631,-0.200268,-0.796225 -378,-0.39304,-0.213419,0.357097 -177,0.176876,0.400283,1.351383 -33,1.639107,0.812341,2.57468 -251,-0.653386,-0.616709,-0.95017 -224,-0.30198,-0.209036,-0.783502 -186,0.930669,-0.393146,-0.062119 -85,1.269946,0.325762,-0.288585 -422,-0.711749,0.400283,-0.237058 -104,-0.839901,-0.4589,-0.672177 -461,5.930172,0.146035,1.08993 -142,-0.663758,0.391516,-0.477519 -380,-0.711046,1.255083,-0.072298 -123,-0.237464,-0.046843,-0.480063 -428,-0.807731,-1.287398,-1.221866 -396,-0.361046,0.45727,0.017398 -66,-0.966647,0.983301,-0.558944 -170,-0.582718,0.268776,-0.812128 -481,-0.088042,-1.138356,-0.717343 -313,-0.725637,-1.015616,-0.583118 -53,0.774214,-0.191501,-0.156268 -296,-0.858535,-1.720497,-1.139994 -114,-1.048038,1.754812,-0.113647 -243,-0.306902,-1.695949,-0.700167 -140,-0.9452,-0.393146,-1.159206 -152,-0.879102,-0.138898,0.145898 -268,-0.497635,-0.296707,-0.46734 -91,-0.087339,-0.292324,-0.34711 -255,-0.095952,0.825491,0.457607 -559,-0.714386,-0.112597,-0.016317 -14,-0.321493,1.43481,3.296698 -100,0.04556,-0.257255,-0.381461 -353,0.29782,1.474263,-0.118736 -543,-0.441206,-1.103288,-0.738972 -165,-0.173125,-1.221645,-0.981659 -43,0.04679,0.904396,0.751503 -201,0.630066,0.251241,0.558117 -108,2.600686,1.65399,2.833589 -328,0.422632,1.167411,0.257223 -439,-0.336962,-1.269864,-0.970527 -162,2.375673,0.501106,0.829112 -325,-0.538243,0.264392,-0.84648 -183,-0.726691,-0.888492,-0.593296 -197,0.610729,-1.935293,-0.368739 -498,1.264672,0.387133,0.347555 -477,-0.284225,-1.688935,-0.341385 -421,-0.125485,-0.051226,0.694887 -374,-0.369132,-0.958629,-0.284132 -470,-0.87084,-0.393146,-0.636553 -382,-0.689424,-1.945375,0.427072 -510,-0.715089,-1.098904,0.159257 -132,0.517559,0.312612,0.325926 -545,-0.265064,-0.472051,-0.652457 -121,1.175019,0.786039,-0.160085 -45,1.206661,1.557551,1.62047 -333,-0.681865,-0.69123,-0.994446 -134,1.247093,0.619463,-0.170263 -182,0.682803,0.3959,0.638907 -519,-0.450875,0.663299,-0.35856 -207,0.579086,-0.932328,-0.672177 -99,-0.095249,0.470421,0.307478 -366,1.389484,-0.200268,0.555572 -180,4.105459,0.650148,0.948707 -318,-1.025712,-0.450133,0.766771 -501,-0.162753,2.061664,0.905449 -229,-0.306902,1.987143,1.781414 -7,0.028859,1.447961,0.724786 -247,-0.424506,-0.305475,2.1033 -494,-0.408333,-0.901643,-0.570395 -174,-0.830233,-1.085753,-1.185478 -340,-0.032492,-0.130131,0.526946 -125,-0.271919,-0.730683,-0.758692 -209,0.091617,-0.445749,-0.22688 -347,0.000381,-0.454517,-0.339476 -332,-0.781363,0.439736,-1.002397 -361,-0.455973,-0.805204,-0.557036 -500,-0.041633,-0.827122,-0.233241 -300,2.061007,0.75097,1.00087 -283,0.26442,0.181104,1.376193 -107,-0.591508,-0.612326,-0.368739 -131,0.484159,0.974533,-0.094562 -303,-0.888068,0.391516,-0.953351 -567,1.653171,1.430427,3.904848 -507,-0.82531,1.48303,-0.325481 -3,-0.550021,3.394275,3.893397 -516,1.076575,0.73782,-0.004231 -143,-0.416244,-0.051226,0.003403 -398,-0.674833,-0.892875,-0.422174 -530,-0.577093,0.110966,-0.438078 -119,0.66874,-1.103288,-0.852841 -259,0.271451,2.451803,1.922 -450,-0.656375,-1.656935,0.544758 -192,-1.013934,-2.682695,-1.443878 -514,0.151913,-0.340543,-0.280951 -241,-0.592739,-1.256713,-1.122819 -402,-0.454742,-1.713045,-0.142909 -148,-0.199845,-0.033692,0.122361 -395,-0.344697,-1.129589,-0.834393 -299,-0.910393,-0.792053,-1.06951 -149,-0.271919,-1.545592,-0.457162 -60,-0.900022,-0.213419,-0.989865 -61,-1.066496,1.382207,-0.537316 -126,-0.056048,0.645764,0.217146 -372,1.150408,-0.577257,0.189156 -82,2.955784,1.09289,2.247704 -425,-0.886486,-0.866574,-1.166203 -337,1.744582,0.764121,1.453165 -386,-0.61647,-1.304933,-0.071025 -184,0.162637,-0.099446,0.481144 -58,-0.450875,-1.326851,-1.223647 -88,-0.537716,-0.086295,-0.050669 -117,0.376926,2.429885,1.232425 -459,-0.932895,-0.936711,-0.912002 -221,-0.341005,0.229323,0.098824 -223,0.364621,1.000835,1.232425 -128,-0.207404,0.273159,0.21651 -294,-0.574632,-0.112597,-0.681083 -541,-0.126013,0.071514,1.055578 -111,-0.620865,-0.160816,-0.186167 -266,-0.801227,-0.485202,-0.01759 -568,-1.075813,-1.859019,-1.207552 -362,-0.528926,-0.112597,-0.44762 -271,-0.743743,0.150419,-0.658818 -187,-0.631237,-0.003007,-0.955896 -155,-0.556174,-0.467667,-0.480063 -5,-0.24432,2.048513,1.721616 -511,-0.211623,-0.809587,-0.974344 -252,1.850057,1.693442,2.170731 -480,-0.585707,-0.50712,-0.167719 -513,-0.032668,-0.441366,-0.391004 -86,-0.126013,-0.077528,-0.360469 -105,-0.093843,2.359748,0.990056 -242,-0.716671,0.102199,1.466524 -312,-0.460192,-0.56849,-0.212884 -473,-0.565491,-1.672278,-1.285861 -364,-0.381613,-0.485202,-0.551311 -565,1.494959,-0.69123,-0.39482 -447,-0.0898,-0.428215,-0.420902 -488,-0.581488,0.886862,-0.677903 -151,-1.108862,1.342755,1.124281 -263,0.189884,-1.050685,-0.467976 -563,1.649655,0.365215,1.0454 -194,-0.168554,-0.033692,1.339296 -433,1.269946,0.290694,0.585471 -244,0.942974,0.610696,0.270582 -489,0.357589,-1.379454,0.240047 -465,-0.258559,-0.537805,1.974164 -483,-0.341181,-0.546572,-0.761237 -397,-0.509062,-1.623181,-0.464796 -52,-0.621217,-0.787669,-1.050935 -147,-0.124606,-1.432057,-0.013773 -27,0.918363,0.062747,-0.270773 -436,-0.445953,-0.480818,-0.566578 -434,-0.181211,-0.463284,-0.631464 -190,-0.207756,0.917547,4.315794 -407,-0.412728,-1.681045,-0.385914 -202,1.943226,0.930698,1.033313 -455,-0.307605,-0.664929,-0.713526 -452,-0.627369,-0.50712,-0.436806 -385,-0.215139,-0.051226,-0.611744 -188,-0.68749,0.141652,-0.981341 -159,-0.721769,-0.669312,-1.089867 -32,0.814646,1.360289,0.64654 -264,0.976374,1.027137,0.01549 -245,-0.839901,0.838642,-0.964802 -535,1.632076,-0.244104,0.376817 -430,-0.084174,0.417818,2.89275 -443,-0.904065,-1.509208,-1.201318 -154,-0.357354,0.676449,-0.18235 -9,-0.297409,2.320295,5.112877 -63,-1.00286,-1.490797,-0.550038 -329,0.166328,0.448503,-0.271409 -536,-0.2677,0.246858,0.121089 -65,0.078257,1.42166,0.555572 -438,-0.231136,-0.901643,-0.891646 -269,-0.826541,0.049596,0.004675 -36,-0.142361,0.536175,1.078479 -98,-0.647058,0.470421,-0.439986 -392,0.841015,1.566318,0.871734 -69,-0.572523,-0.121364,-1.168303 -517,1.345536,0.40905,0.487505 -92,-0.087866,-1.392605,-0.82994 -552,-0.502558,-0.393146,-0.940628 -505,-0.971217,2.990984,0.712699 -566,0.427906,-0.809587,0.350735 -163,-0.575863,0.562476,-0.130186 -358,-1.017099,-1.353152,-0.823579 -442,-0.305847,-1.103288,-0.936175 -15,0.110075,1.553167,2.56641 -503,3.342525,-0.546572,0.688526 -176,-0.862051,-0.099446,0.259131 -327,-0.627897,-1.36192,-1.147374 -1,1.890489,-0.375612,-0.430444 -404,-0.610845,-1.208494,-1.188468 -24,2.345788,2.109883,0.658627 -139,-0.785054,0.189871,-0.458434 -101,-1.222423,1.14111,-0.852841 -350,-0.594321,-1.437317,-1.205517 -90,-0.135154,-0.914793,-0.494058 -191,-0.529278,-1.687182,-1.046355 -383,-0.492362,0.452886,0.668169 -11,0.73554,0.316995,1.950627 -314,-1.125913,0.102199,-1.123391 -352,4.137101,0.904396,2.159281 -235,-0.290202,-0.160816,-0.655002 -305,-0.710343,-1.618359,-0.751695 -375,-0.033546,-0.388763,0.004675 -215,-0.229378,0.597545,1.16245 -466,-0.336611,0.119734,0.640179 -486,-0.087163,-0.796437,-0.300672 -250,1.985416,-0.493969,0.400354 -339,3.560506,0.838642,0.086101 -103,-0.91303,1.03152,-0.153087 -206,-0.965064,0.400283,-0.824215 -345,-0.920238,0.159186,-0.576756 -346,-0.558987,-0.152049,-0.75742 -317,1.062512,0.483572,0.140173 -499,1.545938,0.615079,0.670714 -478,-0.725988,0.124117,-0.33884 -47,-0.213029,2.026595,1.032677 -456,-0.620162,0.360831,-0.325481 -211,-0.587641,-0.191501,-0.421538 -490,-0.452984,-0.296707,-0.469885 -113,-0.889826,-0.103829,-0.314031 -359,-0.77521,0.040829,-0.95017 -344,-0.640202,0.597545,-0.908185 -532,-0.188419,-0.261639,-0.622558 -261,0.593149,-0.366845,-0.672177 -4,1.220724,0.220556,-0.313395 -226,-0.85291,0.075898,-0.884012 -464,-0.339247,-0.182734,-0.367466 -277,0.624792,-0.353694,-0.879559 -257,0.084761,1.93454,1.247056 -260,1.693603,0.869327,0.255951 -301,-0.578851,-1.199727,-0.244691 -537,-0.690654,1.94769,0.450609 -403,-0.526817,-0.664929,-0.371919 -233,1.742824,-0.441366,0.138901 -487,1.51078,0.834259,0.75214 -141,0.619518,-0.042459,-0.195073 -68,-0.977194,0.693984,1.159269 -363,0.225746,0.062747,-0.549402 -379,-0.654793,3.771263,4.348873 -51,-0.393567,-1.028767,-0.611108 -195,-0.492186,-0.993698,-0.659455 -239,0.927153,0.181104,0.758501 -161,1.080091,-0.875341,-0.335023 -169,-0.152733,-0.472051,-0.57612 -212,2.845036,-0.796437,-0.653093 -120,-0.650574,0.983301,-0.097107 -367,-0.449996,0.194255,-0.237058 -171,0.198674,0.338913,-0.634009 -348,-0.711573,0.90878,-0.905004 -409,-0.47531,-0.366845,-0.47561 -112,-0.301628,-1.879621,1.049853 -524,-0.886134,0.417818,-0.19062 -62,0.138729,1.386591,2.356484 -393,2.110228,0.781656,2.01933 -75,0.601939,0.772888,-0.316575 -124,-0.436812,-1.309316,-0.007411 -231,-0.752884,-1.768278,-0.706529 -285,-0.55635,-1.25233,-1.196102 -448,-0.088042,-1.028767,0.067653 -460,1.349052,1.211247,-0.062755 -218,1.983658,0.128501,0.440431 -467,-0.87963,0.281926,-0.819126 -525,-1.063508,1.390974,-0.195709 -80,-0.624908,1.022753,-0.551311 -472,0.045735,-1.133973,0.157985 -549,-0.659188,-0.524654,-0.578665 -527,-0.554943,-0.138898,-0.298127 -168,1.370147,0.229323,0.818934 -351,0.061029,0.992068,1.59248 -57,0.184435,0.194255,1.111558 -560,-0.305671,-0.362461,-0.177261 -424,-0.878399,0.325762,-0.75742 -355,-0.585707,-0.998082,-0.343929 -42,1.354326,-0.33616,3.117943 -217,-0.838319,-1.62625,-0.728794 -286,-0.587992,-0.91041,0.17198 -135,-0.399017,0.417818,-0.64864 -521,3.096417,0.080281,1.046672 -278,-0.248715,-1.199727,-1.132615 -504,-1.020263,2.535091,0.571476 -97,-0.904416,-0.033692,-1.014866 -84,-0.54967,0.233707,-0.343293 -553,-1.027998,-0.967396,-1.089612 -469,-0.619635,2.000293,0.213329 -87,1.305104,-0.327392,0.421983 -22,0.176348,0.290694,2.170095 -19,-0.297761,0.509873,-0.489605 -265,4.485168,0.338913,0.064472 -38,-0.319559,-1.68762,-1.291078 -479,0.103922,0.233707,1.220974 -17,0.763667,2.039746,1.075298 -189,-0.586937,-0.998082,-0.56785 -178,-0.477771,-2.240829,-1.399158 -164,2.676276,-0.419448,0.661808 -451,0.950006,0.895629,-0.443803 -290,-0.199142,-1.426358,-0.044944 -67,-0.72757,-0.147665,-1.03554 -110,-0.902834,0.62823,-0.494694 -556,-0.937465,-0.257255,-0.854113 -150,-0.463884,-0.11698,-0.914547 -321,1.364873,-1.182192,-0.639734 -526,-0.282643,1.316453,0.36982 -18,2.667486,0.825491,0.386359 -446,0.978132,0.338913,0.775677 -432,1.051965,1.496181,0.254042 -523,-0.314109,0.444119,0.014854 -418,-0.551428,-0.042459,-0.595204 -272,2.642876,-0.22657,1.388279 -153,-0.7731,0.075898,-1.0468 -509,0.050658,1.789881,1.542225 -288,-0.778726,-1.296166,-0.445075 -258,0.607213,0.790423,1.672634 -228,-0.451051,-0.423831,0.579746 -476,-0.091558,-0.748217,0.563842 -423,-0.30198,-0.774519,0.397174 -64,0.013566,2.311528,0.965882 -137,-0.735833,-0.586024,-0.569123 -343,1.159197,-0.463284,0.58229 -302,1.433432,0.102199,0.539669 -561,-0.77521,-1.740223,-1.267986 -8,-0.248363,1.662757,1.81831 -502,-0.57762,1.123576,-0.5036 -28,0.682803,1.390974,2.269333 -457,-0.435405,-0.152049,-0.941264 -136,-0.586937,-0.230954,-0.963529 -365,1.581096,0.014527,-0.106013 -384,-0.451578,-0.69123,0.090554 -81,-0.467048,0.930698,1.430264 -306,-0.428374,-0.857807,-0.761237 -282,1.313894,0.851793,0.767407 -475,-0.483045,0.010144,0.042844 -118,0.688077,2.329063,1.515507 -55,-0.658133,-0.327392,-1.062767 -557,-0.966822,-1.098904,-1.162132 -284,-0.500975,-1.451345,-0.143545 -256,1.837752,-0.187118,1.772508 -426,-0.773804,0.014527,0.288394 -387,-0.237816,-2.083458,-0.833121 -497,-0.480408,-0.209036,-0.023315 -298,-0.107027,-1.662195,-0.238966 -157,0.266178,-1.956334,-0.529682 -249,-0.683447,0.28631,-0.611108 -342,-0.774507,0.413434,-0.211612 -508,0.083706,0.132884,-0.751695 -267,-0.385129,-1.396988,-0.516959 -338,-0.87295,0.343297,-0.725613 -360,-0.518379,-1.728826,-1.342223 -12,0.793551,-1.256713,0.865372 -210,1.363115,-0.638627,0.240047 -533,1.343778,-0.993698,-0.005503 -167,0.666982,-0.682463,-0.269501 -79,-0.454391,-0.152049,-0.255506 -369,2.366883,-0.130131,0.853922 -437,-0.229554,-0.564106,-0.821034 -485,-0.527344,-0.651778,0.965882 -408,0.823436,0.693984,0.758501 -308,-0.319735,-1.847183,-1.24623 -414,0.089332,-0.770135,-0.989865 -41,-0.644421,2.565776,0.098824 -276,-0.742864,-0.283557,-1.150045 -146,-0.507831,0.268776,0.985603 -405,-0.71755,0.172337,-0.571667 -35,0.612486,1.049055,0.822115 -13,-0.007178,-0.844656,-0.393548 -166,-0.820212,0.492339,-0.817853 -173,-0.850976,-0.472051,-1.093302 -531,-0.580082,0.992068,0.268037 -388,-0.756928,-0.97178,0.169436 -205,0.191466,0.733436,0.5015 -6,1.27522,0.51864,0.021215 -144,-0.772397,-1.085753,-0.839482 -540,-0.743216,0.093432,-0.270137 -30,1.387726,0.733436,1.090566 -435,-0.019835,1.268234,0.652266 -31,0.014269,1.37344,2.056226 -203,0.478885,3.955374,1.696171 -59,-1.064738,1.794265,-0.829304 -95,1.528359,-0.586024,0.633818 -37,-0.588344,-1.549975,-1.323648 -331,-0.432944,-0.156432,0.451882 -237,1.528359,-0.419448,-0.147362 -10,0.473611,-0.625477,-0.630828 -389,0.760151,-0.318625,-0.08184 -324,-0.522949,-0.296707,-0.391004 -179,-0.544747,-0.708765,-1.271103 -220,-0.30655,-0.05561,-0.043671 -50,-0.640026,-1.046301,-1.069447 -401,-0.511699,0.220556,-0.615561 -368,3.977131,0.172337,-0.581845 -29,0.608971,-0.301091,0.171344 -78,1.305104,1.382207,2.303684 -204,-0.356299,0.448503,-0.104741 -214,-0.121794,1.03152,0.96461 -127,1.155681,-1.326851,-0.177261 -46,-1.122222,-0.11698,-0.754239 -554,-0.5008,-0.423831,-0.586935 -449,2.006511,0.194255,0.355188 -564,2.015301,0.378365,-0.273318 -297,-0.574808,-0.818354,-1.110223 -491,0.579086,-1.4794,-0.982868 -492,0.958795,-0.064377,-0.137184 -181,2.124291,0.733436,3.207003 -427,-0.68749,-0.090679,-0.538588 -471,-0.550197,-1.239179,-0.998771 -515,-0.706651,0.698367,-0.616197 -445,-0.644597,-0.05561,-0.458434 -506,-0.642136,0.343297,-0.144817 -93,-0.318504,0.067131,-0.5036 -453,-0.22973,0.102199,-0.677266 -357,-0.327294,-0.748217,-0.976252 -544,-0.33749,-0.261639,-0.321664 -26,0.028684,0.882478,2.608395 -356,-0.50871,0.084665,0.073378 -227,-0.127243,-0.822738,0.689798 -377,-0.351553,-0.945479,-0.690625 -234,-0.957505,0.790423,-1.012194 -341,-0.92639,-0.39753,0.555572 -129,1.245335,-0.213419,0.838655 -289,-0.740579,-0.901643,-0.999916 -539,-1.154919,1.193713,0.331651 -145,-0.652156,0.439736,-0.016317 -138,0.159648,0.382749,-0.240875 -410,-0.631588,0.56686,-0.585662 -56,2.222734,1.316453,0.616006 -216,-0.614185,0.356447,0.320201 -281,-0.60979,-1.261097,-1.076762 -16,0.452516,0.615079,-0.427264 -444,0.723235,-0.266022,0.078468 -547,-0.91971,0.601928,-0.188711 -270,-0.344521,-2.047074,-1.297121 -44,-0.245902,0.786039,0.866009 -34,0.66874,0.536175,2.074674 -371,-0.108082,-0.866574,-0.512506 -310,-0.698741,-0.441366,-0.925997 -538,-1.112026,-0.296707,-1.08694 -72,1.407063,1.145493,3.086136 -73,-0.11986,0.382749,0.635726 -279,-0.370187,-0.607942,-0.520776 -196,-0.129529,1.811799,0.368547 -74,-0.473728,-0.647394,-0.445075 -23,3.048953,0.338913,0.036482 -334,-0.591156,-0.445749,-1.041647 -400,0.74433,2.407967,2.146558 -431,-0.641257,0.553709,0.05493 -254,1.918616,0.759738,0.393357 -48,-0.523828,0.746587,-0.245964 -130,-0.589574,0.452886,0.02694 -555,-0.919359,0.264392,-0.529682 -520,-0.973854,2.307145,-0.283496 -122,2.096165,1.632072,1.082296 -496,-0.433999,0.917547,0.826568 -225,-0.012979,-0.11698,-0.647368 -326,-0.22973,-0.187118,-0.912002 -106,-0.630885,1.597003,0.074651 -381,-0.762026,-1.002465,-0.356652 -415,-0.628776,0.448503,-0.226243 -71,-1.048038,-0.511503,-0.067845 -558,-0.258559,-1.304933,0.399718 -21,-0.994422,0.001377,-0.887193 -316,-0.646003,-1.414523,-1.278291 -463,-0.677646,0.080281,-0.46734 -454,-0.434351,-0.432599,-0.652457 -413,-0.023702,-1.08137,0.510406 -330,0.332978,0.487955,1.231153 -468,0.978132,-0.511503,1.426448 -322,-0.49412,0.978917,-0.198253 -307,-1.046104,-1.479838,-1.284653 -373,1.87291,1.044671,0.325926 -304,-0.686963,-0.787669,-0.479427 -529,-0.581312,0.864944,-0.579301 -40,-0.162929,-1.006849,-0.317847 -115,-0.523125,0.772888,-0.091382 -2,1.456285,0.527407,1.082932 -39,-0.24643,1.255083,1.070209 diff --git a/examples/data/breast_homo_guest.csv b/examples/data/breast_homo_guest.csv deleted file mode 100644 index 4163c1279..000000000 --- a/examples/data/breast_homo_guest.csv +++ /dev/null @@ -1,228 +0,0 @@ -id,y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15,x16,x17,x18,x19,x20,x21,x22,x23,x24,x25,x26,x27,x28,x29 -133,1,0.254879,-1.046633,0.20965599999999998,0.074214,-0.441366,-0.377645,-0.48593400000000003,0.347072,-0.28757,-0.733474,0.44951199999999997,-1.247226,0.413178,0.30378099999999997,-0.123848,-0.184227,-0.21907600000000002,0.26853699999999997,0.015996,-0.789267,-0.33736,-0.728193,-0.442587,-0.272757,-0.6080180000000001,-0.577235,-0.5011260000000001,0.143371,-0.46643100000000004,-0.5541020000000001 -273,1,-1.1429280000000002,-0.781198,-1.166747,-0.923578,0.6282300000000001,-1.021418,-1.111867,-0.9595229999999999,-0.09667200000000001,-0.121683,-1.245485,-0.842317,-1.255026,-1.038066,-0.42630100000000004,-1.088781,-0.976392,-0.898898,0.9834959999999999,0.045702,-0.49363900000000005,0.34862,-0.552483,-0.526877,2.253098,-0.82762,-0.7807390000000001,-0.37699699999999997,-0.310239,0.17630099999999999 -175,1,-1.451067,-1.406518,-1.456564,-1.0923370000000001,-0.708765,-1.168557,-1.305831,-1.745063,-0.49949899999999997,-0.30289299999999997,-1.549664,-1.126219,-1.5466520000000001,-1.216392,-0.35442399999999996,-1.167051,-1.114873,-1.2618200000000002,-0.32719299999999996,0.6297550000000001,-0.6668810000000001,-0.779358,-0.708418,-0.637545,0.7103689999999999,-0.976454,-1.057501,-1.9134470000000001,0.795207,-0.149751 -551,1,-0.879933,0.42058900000000005,-0.8775270000000001,-0.7804840000000001,-1.037534,-0.48388000000000003,-0.5554979999999999,-0.7685810000000001,0.43396,-0.200928,-0.851273,0.733108,-0.843535,-0.786363,-0.049836,-0.42453199999999996,-0.509221,-0.679649,0.797298,0.385927,-0.451772,0.45385200000000003,-0.431696,-0.49475399999999997,-1.182041,0.28122800000000003,0.084759,-0.25242,1.038575,0.351054 -199,0,0.42675799999999997,0.723479,0.316885,0.287273,1.0008350000000001,0.9627020000000001,1.077099,1.0535860000000001,2.996525,0.9616959999999999,0.091654,0.21649899999999997,0.10383900000000001,-0.034667,0.16793,0.308132,0.366614,0.280661,0.505223,0.264013,-0.707304,-1.026834,-0.702973,-0.460212,-0.999033,-0.531406,-0.39436,-0.72883,-0.644416,-0.6880029999999999 -274,0,0.963102,1.467675,0.829202,0.772457,-0.038076,-0.468613,-0.307946,-0.015321000000000001,-0.641864,-0.247477,1.080023,1.20783,0.956888,0.978402,-0.555822,-0.645696,-0.399365,-0.038153,-0.998966,-1.091216,0.057848000000000004,0.392164,-0.050026999999999995,0.12041400000000001,-0.5323479999999999,-0.770613,-0.519694,-0.531097,-0.769127,-0.394858 -420,1,-0.662496,0.21214899999999998,-0.620475,-0.632995,-0.327392,-0.385278,-0.077665,-0.730362,0.217178,-0.061279999999999994,-0.7263069999999999,-0.05809500000000001,-0.7319100000000001,-0.697343,-0.7757229999999999,-0.513983,-0.426233,-0.893482,0.800949,-0.018090000000000002,-0.42867299999999997,0.40486500000000003,-0.32675,-0.44085,0.07901,-0.279903,0.41699200000000003,-0.48616499999999996,-0.22548400000000002,-0.17244600000000002 -76,1,-0.45334300000000005,-2.147457,-0.473631,-0.483572,0.5580930000000001,-0.740244,-0.89617,-0.617229,-0.308601,-0.666975,-0.169639,-1.9430189999999998,-0.167192,-0.27215,2.3299369999999997,0.006804000000000001,-0.251467,0.429234,2.1591,0.5120939999999999,0.017786,-0.368046,-0.105966,-0.169129,2.11976,0.162743,-0.672216,-0.577002,0.626908,0.896114 -315,1,-0.606584,-0.971725,-0.678558,-0.591332,-0.9630129999999999,-1.302401,-1.212855,-1.3211540000000002,-1.591501,-1.2305540000000001,-0.46501400000000004,-0.567723,-0.526371,-0.49285200000000007,-0.8006310000000001,-1.250816,-1.058714,-1.0961450000000001,-2.1782209999999997,-0.8601469999999999,-0.8430110000000001,-0.9103530000000001,-0.90049,-0.608283,-0.7043550000000001,-1.255622,-0.9706290000000001,-1.363557,-0.800607,-0.9270579999999999 -399,1,-0.583805,-0.193332,-0.633283,-0.560041,-0.34931,-0.519504,-0.6106689999999999,-0.9295260000000001,-0.196974,-0.151608,-0.660984,-0.47231300000000004,-0.688248,-0.634204,-0.390718,-0.7963600000000001,-0.75668,-0.8393139999999999,0.129175,-0.369656,-0.221505,-0.139439,-0.317344,-0.336122,-0.526014,-0.326291,-0.368166,-1.0378399999999999,-0.698901,-0.273818 -238,1,-0.10751500000000001,2.420311,-0.141817,-0.204943,-1.063835,-0.074206,0.164131,-0.49358900000000006,-1.635181,-0.33170900000000003,0.026330000000000003,1.9920509999999998,0.02393,-0.088136,-1.005588,-0.008357,0.26994,-0.124821,-1.714551,-0.213719,-0.251822,2.0087450000000002,-0.376748,-0.22831300000000002,-0.24466999999999997,0.166096,0.219045,-0.273508,-1.052451,-0.07788300000000001 -246,1,-0.482335,0.348938,-0.565371,-0.489725,-0.9761639999999999,-0.6581819999999999,-0.20335999999999999,-0.988301,-0.216387,-0.663096,-0.263364,-0.432753,-0.322891,-0.322206,-1.7229349999999999,-1.1200510000000001,-0.570489,-0.9767959999999999,-1.1851639999999999,-0.9140159999999999,-0.87405,0.696974,-0.986625,-0.589142,-0.260004,-0.5470550000000001,-0.036596,-1.040273,-0.111671,-0.5843619999999999 -253,0,0.7415229999999999,-0.095626,0.704101,0.6001810000000001,0.404667,-0.087565,0.314773,1.082516,0.383809,-0.156041,0.9010940000000001,-0.5142,0.8662700000000001,0.777324,0.31595500000000004,-0.004567,0.47458599999999995,0.892752,0.005043,-0.945203,-0.346022,-0.653261,-0.333185,-0.147567,-0.761357,-0.583942,-0.284279,0.133639,-0.799396,-0.541998 -550,1,-0.954483,-0.14773599999999998,-0.9883299999999999,-0.8232010000000001,-1.414523,-1.150045,-1.305831,-1.745063,-0.716282,-0.998915,-0.9279569999999999,0.509709,-0.966282,-0.8372729999999999,-1.569218,-1.1763370000000002,-1.114873,-1.2618200000000002,-0.5499,-0.47030600000000006,-0.320758,0.158114,-0.371797,-0.432709,0.8460409999999999,-0.806941,-1.057501,-1.9134470000000001,1.149967,-0.592683 -208,1,-0.356014,0.567149,-0.23176999999999998,-0.424155,0.11096600000000001,1.182806,0.211146,-0.030548000000000002,1.985412,1.310816,-0.288925,0.756379,-0.203852,-0.356904,0.273255,0.8330879999999999,-0.021963,0.054189,0.140128,1.4604709999999999,-0.765413,-0.534421,-0.680696,-0.555479,-0.596684,0.27563899999999997,-0.21995399999999998,-0.566458,0.556683,0.152472 -185,1,-0.9109950000000001,-0.732345,-0.949311,-0.77978,0.864944,-0.969255,-1.272632,-1.586402,0.052164,-0.386571,-1.1494879999999998,-0.9726319999999999,-1.161936,-0.959569,-0.26261999999999996,-1.087644,-1.0948229999999999,-1.199811,-0.39655999999999997,-0.328545,0.069759,0.092797,-0.09210499999999999,-0.30598000000000003,2.449773,-0.753287,-1.004549,-1.523495,0.585742,-0.122895 -156,0,0.869914,-0.092369,0.7636729999999999,0.7408140000000001,0.413434,0.6077359999999999,0.41312200000000004,0.561767,-0.7081930000000001,-0.36385,1.00902,0.337507,1.047507,0.87829,1.077425,1.178005,1.2140739999999999,1.456866,0.581892,-0.161268,1.4658010000000001,0.332291,1.323683,1.178696,0.665367,1.344806,0.668654,1.072832,-0.334455,0.443725 -0,0,1.8866900000000002,-1.359293,2.303601,2.0012369999999997,1.3076860000000001,2.6166650000000002,2.109526,2.296076,2.750622,1.9370150000000002,1.097064,-2.0733349999999997,1.2699340000000001,0.984375,1.568466,3.283515,2.652874,2.532475,2.217515,2.255747,2.489734,-0.565265,2.833031,2.487578,-0.214002,1.316862,0.7240260000000001,0.66082,1.148757,0.907083 -70,0,1.779007,0.14701199999999998,1.746605,1.732277,-0.572873,-0.131459,-0.016736,0.978975,-0.565828,-1.000578,1.3668770000000001,0.470149,1.302886,1.351264,-0.44622700000000004,-0.027308999999999996,0.241064,0.78906,-0.838325,-1.160679,1.384594,-0.760851,1.296951,1.2257799999999999,-0.8656950000000001,-0.5006659999999999,-0.305168,0.308825,-0.809083,-0.793157 -293,1,-0.664567,0.011851,-0.68243,-0.637741,0.198638,-0.499147,-0.674477,-0.353352,0.32395100000000004,-0.76894,-0.646783,-0.425771,-0.676715,-0.631929,-0.899551,-0.9081739999999999,-0.777395,-0.673717,0.231402,-0.800607,-0.715245,0.038367,-0.8079189999999999,-0.582101,0.18468099999999998,-0.5856180000000001,-0.589324,-0.5220130000000001,-0.317504,-0.760627 -287,1,-0.548601,-1.650784,-0.591583,-0.533673,-1.587236,-0.8878290000000001,-0.7368439999999999,-0.928004,-0.9573309999999999,-0.8199219999999999,-0.351408,-1.435719,-0.415157,-0.39529899999999996,-1.907966,-1.270715,-0.8311299999999999,-0.959772,-1.732806,-0.990566,-0.9094200000000001,-1.356863,-0.866828,-0.608503,-0.7700239999999999,-0.672247,-0.5097470000000001,-0.955599,-0.5269699999999999,-0.6482859999999999 -222,1,-1.0559530000000001,-0.462024,-1.052072,-0.887716,0.360831,-0.70144,-0.9905370000000001,-0.89618,0.249533,0.223003,-1.1210870000000002,-0.409482,-1.105917,-0.972083,0.693131,-0.366161,-0.8929010000000001,-0.7678649999999999,0.359185,0.890594,-0.571959,0.000266,-0.606442,-0.556359,0.286019,-0.641508,-0.774539,-0.676761,0.705609,-0.012824 -262,0,0.853348,0.254488,0.912602,0.728509,-0.8315049999999999,0.20633200000000002,-0.20335999999999999,0.581561,0.268947,-0.504606,0.8982540000000001,0.6609689999999999,0.9239360000000001,0.8330690000000001,-0.45334399999999997,0.43510699999999997,0.102583,0.674535,1.082072,-1.156426,1.550617,0.755033,1.6236709999999999,1.113351,-0.10799700000000001,1.9187919999999998,0.480322,1.572438,0.8387959999999999,0.772804 -309,1,-0.318739,-1.3478940000000001,-0.396188,-0.36596799999999996,-1.348769,-1.24553,-1.218324,-1.207259,-1.284123,-1.005565,-0.30596599999999996,-1.26817,-0.38138099999999997,-0.353491,-0.913784,-1.269578,-1.057635,-1.0340850000000001,-1.309296,-1.079875,-0.02769,-0.704607,-0.148043,-0.161208,-0.9643649999999999,-1.160666,-0.963633,-1.131435,-0.765494,-0.575662 -534,1,-0.962766,0.135613,-0.918334,-0.8316389999999999,0.45726999999999995,-0.02077,-0.28731599999999996,-0.24356799999999998,-0.9896870000000001,-0.064605,-0.8995559999999999,-0.388538,-0.8723690000000001,-0.822768,0.036274,-0.129268,-0.45410500000000004,-0.542683,-0.70324,0.181792,-0.918443,0.664316,-0.842076,-0.6654869999999999,0.82004,0.463987,0.40174,0.269895,-0.8865729999999999,-0.098687 -54,0,0.379129,0.979143,0.31092800000000004,0.262662,0.28631,-0.308942,-0.0047420000000000006,0.584607,-0.365223,-0.289039,0.27626300000000004,0.635371,0.21793600000000002,0.164705,-0.41278000000000004,-0.635462,-0.45536000000000004,-0.401848,-0.7141930000000001,-0.8445530000000001,-0.34169099999999997,-0.694809,-0.380708,-0.22941399999999998,-0.7886920000000001,-0.848299,-0.525994,-0.41755,-1.1614209999999998,-0.819256 -172,0,0.522016,-1.406518,0.528365,0.389232,0.9087799999999999,0.661808,1.491126,1.036837,0.509996,0.9450709999999999,0.378508,-1.721948,0.43377299999999996,0.23324699999999998,2.087974,0.9695389999999999,1.436297,1.56778,0.563638,1.1188280000000002,0.056764999999999996,-1.013408,-0.030226,0.094672,-0.549349,-0.12676500000000002,0.369246,0.22771999999999998,-0.317504,0.141124 -484,1,0.153409,-1.868995,0.15604200000000001,-0.046202999999999994,0.952616,0.277579,0.6150979999999999,0.46583900000000006,-0.556121,-0.11835799999999999,0.455192,-1.8638990000000002,0.44613,0.262542,0.565034,0.48438100000000006,0.380424,0.34024499999999996,-0.10083400000000001,-0.029431,-0.87405,-1.505458,-0.852967,-0.582321,-0.335673,-0.40677199999999997,0.048618,-0.18267,-1.064558,-0.582092 -102,1,-0.606584,1.1664139999999998,-0.675579,-0.585004,-0.879725,-1.053734,-0.756514,-0.613574,-0.334485,-0.840426,-0.5530579999999999,0.28631100000000004,-0.607516,-0.557982,-1.155035,-1.2121549999999999,-0.815688,-0.805266,-0.265127,-0.854476,-0.767939,0.642544,-0.833166,-0.5645,-0.653686,-1.0831469999999999,-0.703052,-0.810908,-0.735225,-0.8559459999999999 -458,1,-0.399502,1.010084,-0.482567,-0.44314,-0.46328400000000003,-0.9221799999999999,-1.0917649999999999,-0.8434959999999999,-0.962184,-1.1657170000000001,-0.32016700000000003,1.359089,-0.3855,-0.38306999999999997,-0.9016860000000001,-1.0160069999999999,-0.96346,-0.80733,-0.527994,-1.17769,-0.5163770000000001,0.027481000000000002,-0.598521,-0.42126800000000003,-0.329006,-0.922409,-0.869136,-0.8856870000000001,-1.016127,-0.8627549999999999 -495,1,-0.053674,0.45641499999999996,-0.100117,-0.170488,-0.472051,-0.7345189999999999,-0.490252,-0.196518,-0.8602639999999999,-0.995036,0.21094000000000002,0.21417199999999997,0.170979,0.073978,-0.034891000000000005,-0.395915,-0.25812199999999996,0.01524,-1.1851639999999999,-0.753827,-0.6239319999999999,0.760476,-0.628718,-0.406967,-0.5420149999999999,-0.657716,-0.34363,0.005493,-0.115303,-0.7500359999999999 -158,1,-0.648001,-1.183422,-0.690472,-0.611372,-0.213419,-0.833757,-0.891517,-0.6753939999999999,-0.625686,-0.275185,-0.58714,-1.524147,-0.623168,-0.586707,-0.23130799999999999,-0.9841690000000001,-0.867289,-0.755484,-0.809117,-0.5284270000000001,-0.8047529999999999,-0.886041,-0.8391059999999999,-0.595962,-0.504347,-0.876859,-0.7809050000000001,-0.895906,-0.713431,-0.47845200000000004 -160,1,-0.610726,0.086759,-0.546606,-0.59186,0.150419,-0.413905,-0.367435,-0.540791,0.43234300000000003,-0.225865,-0.675185,0.207191,-0.6536489999999999,-0.668618,0.892395,0.184948,-0.255736,-0.297641,0.662213,0.24558400000000002,0.34875100000000003,0.863894,0.5247029999999999,-0.043939,0.797373,-0.079818,0.324152,-0.044792,1.6258059999999999,0.47890299999999997 -292,1,-0.523751,-0.9359,-0.549585,-0.5189060000000001,0.698367,-0.301944,-0.230706,-0.13713399999999998,0.775312,0.659126,-0.334368,-0.7608689999999999,-0.36367,-0.40155599999999997,0.294605,-0.4721,-0.342114,-0.39256199999999997,-0.297985,0.26968400000000003,-0.7065819999999999,-0.822358,-0.809404,-0.49871400000000005,0.5613630000000001,-0.304494,-0.28328400000000004,-0.077234,0.69108,0.352188 -493,1,-0.637646,-1.517252,-0.715492,-0.609263,-1.664826,-1.205453,-1.2255200000000002,-1.33699,-1.004247,-0.757302,-0.47353500000000004,-1.503204,-0.541199,-0.5050819999999999,-1.611206,-1.211208,-1.024816,-0.9654469999999999,-0.7251449999999999,-0.378161,-0.279974,0.488324,-0.375263,-0.34624299999999997,1.116386,-0.863389,-0.844765,-0.6328020000000001,0.986511,0.38812199999999997 -394,1,-0.561026,0.019993,-0.563882,-0.5644359999999999,0.474804,-0.489605,-0.536788,-0.790964,0.23982699999999998,-0.727932,-0.5757800000000001,-0.365268,-0.572504,-0.593533,0.46540200000000004,-0.128131,-0.514369,-0.403912,0.45776000000000006,-0.168356,-0.436974,0.789506,-0.493575,-0.398607,0.36835500000000004,-0.5124029999999999,-0.446417,-0.694766,-0.047499,-0.755332 -528,1,-0.341518,-1.676839,-0.379508,-0.399544,0.308228,-0.749786,-0.557897,-0.19956300000000002,-1.19838,-0.632617,-0.053194000000000005,-1.424083,-0.06833600000000001,-0.17260599999999998,2.023925,-0.128699,0.153179,0.44445200000000007,0.600147,0.251255,0.5086390000000001,2.5730060000000003,0.606383,0.096872,0.999714,0.39077199999999995,0.521768,2.714394,-0.410734,0.655546 -335,0,0.9775969999999999,1.216895,1.070467,0.846289,0.549325,-0.311486,0.574799,1.036837,-0.44934799999999997,-0.440878,0.832931,0.39801,0.816841,0.750021,1.105891,0.02386,0.778417,1.300555,-0.308938,-0.29594000000000004,1.483125,1.6549479999999999,1.589019,1.030404,-0.19533399999999998,-0.419626,0.43522799999999995,0.915489,-0.27270500000000003,-0.023415000000000002 -311,1,0.039513,-0.639524,-0.106074,-0.069935,-1.370687,-1.166649,-1.0781399999999999,-0.859941,-0.599801,-1.495995,0.137096,-0.8376620000000001,0.029285000000000002,0.028472000000000004,-1.4361389999999998,-1.311272,-0.9332020000000001,-0.777667,-0.655777,-1.452705,-0.321841,-0.554016,-0.45149700000000004,-0.251635,-0.67002,-1.087451,-0.88094,-0.887634,-0.678318,-1.096968 -25,0,1.238521,-0.696519,1.3444969999999998,1.020322,0.97015,0.8946350000000001,0.542655,2.1377200000000003,1.8851099999999998,1.216609,0.8556520000000001,-0.672441,0.9898399999999999,0.733241,1.582699,2.335941,1.68363,2.351917,4.484751,1.606484,2.3128830000000002,-0.43699099999999996,2.183056,1.5635059999999998,0.329354,0.699282,0.179919,1.974718,0.307261,1.380276 -522,1,-0.8985700000000001,0.12258499999999999,-0.919823,-0.781714,-0.9454790000000001,-1.12619,-1.184309,-1.3138459999999998,-0.556121,-0.43312,-0.8143520000000001,0.125743,-0.8513620000000001,-0.758776,-0.8006310000000001,-1.1410870000000002,-1.0512569999999999,-1.1157219999999999,-0.637523,0.089648,-0.977273,-0.242856,-0.9330629999999999,-0.6821649999999999,-0.947031,-1.0942129999999999,-0.9299790000000001,-1.254065,-0.686793,-0.567341 -512,0,0.029158999999999997,0.64857,0.17987,-0.06360700000000001,1.097274,0.8354739999999999,1.143785,1.377912,1.106957,1.493688,-0.20656100000000002,0.28631100000000004,-0.137124,-0.27926,1.013376,0.806556,0.6993199999999999,0.846065,1.1112790000000001,1.481735,-0.05259400000000001,-0.519362,0.11234300000000001,-0.14668699999999998,-0.5423479999999999,-0.158063,0.08707999999999999,0.250429,-0.422842,0.07946900000000001 -230,0,0.687682,-0.128194,0.7815439999999999,0.5421699999999999,1.662757,0.885093,1.101567,2.127061,0.336894,0.36930100000000005,0.830091,-0.048787,0.8827459999999999,0.6829,1.2624549999999999,1.001757,1.2831270000000001,1.5497239999999999,1.166043,0.064131,-0.394386,-0.975851,-0.352986,-0.18387,-0.503013,-0.301699,-0.044554,0.331534,-1.062137,-0.551832 -548,1,-1.105653,-0.014204,-1.136664,-0.9077559999999999,-0.5465720000000001,-1.010222,-0.8572620000000001,-1.159448,-0.56421,-0.262993,-1.262242,0.011717,-1.2735610000000002,-1.0500120000000002,-0.814864,-1.024157,-0.8214629999999999,-1.01381,-0.845627,-0.063453,-0.395108,0.26516,-0.401994,-0.486174,0.133012,-0.796322,-0.282621,-0.353802,0.18012899999999998,0.135829 -213,0,0.37291599999999997,0.38964899999999997,0.39135,0.24684099999999998,-0.353694,-0.476882,0.038915,-0.07166,-2.099484,-0.873676,0.935176,1.4591530000000001,0.928055,0.8336370000000001,0.301721,0.194424,0.996873,0.439809,-1.838683,-0.586549,0.449087,0.816721,0.44599300000000003,0.400275,8.029999,3.357389,3.7104790000000003,4.456526,0.146227,3.315409 -390,1,-1.012466,-1.632871,-1.0136479999999999,-0.854492,0.084665,-0.56785,-0.892524,-0.72549,0.058635,-0.372717,-1.098366,-1.645155,-1.079967,-0.947908,0.256176,-0.548095,-0.873441,-0.7541939999999999,-0.042419,0.410026,-0.7726310000000001,-1.2140739999999999,-0.751486,-0.626104,-0.45301099999999994,-0.6610689999999999,-0.775799,-0.789497,-0.140729,-0.5385939999999999 -275,1,-0.801242,-1.088973,-0.828082,-0.71755,0.154802,-1.085159,-0.962664,-0.382891,-1.101313,-1.30869,-0.635423,-0.449042,-0.64953,-0.623681,1.8602450000000001,-0.6110140000000001,-0.370489,0.647967,0.742534,-0.5737909999999999,0.851874,1.9525,0.571731,0.187079,2.37977,-0.591766,-0.637401,1.184757,0.755252,-0.299917 -219,0,2.408538,3.2133599999999998,2.172543,2.806362,0.369598,0.988783,0.61078,0.729259,-0.303748,-0.458057,1.534446,3.067156,1.4841229999999999,1.615766,-0.865392,0.164101,0.322671,0.45012700000000005,-1.40057,-1.370484,1.2055770000000001,0.188958,0.918747,1.5305030000000002,-0.50068,0.05375800000000001,-0.174198,-0.164827,-0.8744649999999999,-0.582092 -236,0,3.052564,1.4383629999999998,2.941018,3.627307,0.6896,1.007232,1.486328,2.203194,0.327187,0.156504,2.579618,1.7872689999999998,2.5344729999999998,2.88708,-0.0904,1.210223,1.3333469999999998,1.928896,0.355534,0.041449,2.356193,-0.45966999999999997,2.1687,2.540382,-0.204335,0.176156,0.433571,0.8700700000000001,-0.562083,-0.28062600000000004 -336,1,-0.5278930000000001,-1.4276879999999998,-0.592179,-0.535431,-0.796437,-0.361105,-0.61019,-0.8543069999999999,-0.7583439999999999,0.93953,-0.32300700000000004,-1.177414,-0.32495100000000005,-0.39985,-0.123848,-0.088901,-0.645568,-0.720662,-0.582759,1.3584040000000002,-0.8076399999999999,-1.044433,-0.962418,-0.57044,-0.603351,-0.135708,-0.025323,-0.726884,-0.502754,0.720983 -20,1,-0.36636799999999997,-0.844707,-0.332744,-0.43962399999999996,-0.051226,0.14844300000000002,-0.399099,-0.6361100000000001,0.45822700000000005,-0.11725,-0.297446,-0.833008,-0.261106,-0.38363800000000003,0.792763,0.429422,-0.541362,-0.45962700000000006,0.567289,0.753087,-0.793925,-0.8512059999999999,-0.73416,-0.56472,-0.981366,-0.363178,-0.49449399999999993,-0.860707,-0.455534,-0.5181680000000001 -193,0,-0.128223,2.224898,-0.16564500000000001,-0.196329,2.022211,1.376193,0.817074,0.476498,0.508378,2.02568,-0.507616,1.7616720000000001,-0.44563800000000003,-0.504797,0.500985,0.586719,0.247342,-0.08509800000000001,0.479666,0.9317049999999999,0.000462,1.074358,-0.11091600000000001,-0.129746,0.685701,0.7249909999999999,0.190198,0.22934200000000002,-0.213376,0.710014 -200,1,-0.378793,0.436874,-0.4501,-0.425737,0.461654,-0.318484,-0.645212,-0.10059,-0.376548,-0.12223699999999998,-0.538858,0.062913,-0.553145,-0.551441,-0.035602999999999996,-0.44481000000000004,-0.589196,-0.202461,0.6111,-0.378161,-0.186856,0.19802899999999998,-0.276256,-0.28815799999999997,0.15768,-0.429687,-0.592971,-0.06425700000000001,-0.667421,-0.172825 -83,0,0.840923,1.146872,1.013874,0.7337819999999999,0.29946100000000003,0.174525,-0.139073,1.058154,-0.954095,0.447992,1.41232,1.629029,1.529432,1.356952,1.789079,1.416794,1.317025,2.527316,-0.648476,1.338557,0.410829,3.07195,1.4528860000000001,0.58883,0.168014,1.957356,-0.34993,1.076077,1.212928,2.4946040000000003 -376,1,-1.12222,-0.46528100000000006,-0.9159510000000001,-0.9293790000000001,-0.792053,0.684709,1.587076,0.485634,-0.49141,1.997972,-1.0103209999999998,0.21649899999999997,-0.8987299999999999,-0.9004120000000001,-0.400681,1.1685299999999998,1.7476610000000001,0.27060100000000004,1.374147,3.07654,-1.059924,0.025667000000000002,-0.24903000000000003,-0.728456,0.48602700000000004,2.847676,4.032102,2.821453,-0.529391,3.179239 -295,1,-0.331164,-1.424431,-0.389933,-0.385832,-0.673696,-0.9355389999999999,-1.126787,-0.8616159999999999,-0.12579200000000001,-0.886975,-0.101476,-1.4008129999999999,-0.16101400000000002,-0.20531300000000002,-0.311725,-0.7984439999999999,-0.9814139999999999,-0.767349,-0.8018149999999999,-0.521339,-0.671573,-0.9475469999999999,-0.6866369999999999,-0.497174,-0.8976959999999999,-0.9682940000000001,-0.9158540000000001,-0.805718,0.1208,-0.7538189999999999 -77,0,1.267513,-1.102,1.275989,1.282251,0.676449,1.96653,0.510512,1.455568,1.375509,1.488146,1.114105,-0.730617,1.162839,0.9985950000000001,0.721598,2.089571,0.999384,1.523931,1.242713,0.557457,2.0768400000000002,-1.2089940000000001,1.7053509999999998,2.078346,0.299686,1.839429,0.48695299999999997,1.444293,0.649913,1.2327569999999999 -441,0,0.8512770000000001,1.593064,0.760694,0.709172,0.492339,1.004687,1.1102020000000001,0.902842,-0.648335,-0.249694,0.892574,1.426574,0.841556,0.779031,-0.9287290000000001,0.124303,0.396746,0.217723,-1.258183,-1.2372299999999998,0.378346,0.838493,0.206398,0.396975,0.356021,0.9837600000000001,0.5811189999999999,0.9122450000000001,-0.557239,-0.021146 -109,1,-0.674921,0.56552,-0.693153,-0.6372140000000001,1.6452229999999999,-0.220518,0.190997,-0.484605,-0.116085,0.24239899999999998,-0.7916300000000001,0.45851400000000003,-0.8027569999999999,-0.734885,-0.624141,-0.731356,-0.470426,-0.771992,-1.1851639999999999,0.35332199999999997,-0.616352,-0.418666,-0.628223,-0.526437,0.690702,-0.553761,-0.247475,-0.8696280000000001,-0.58872,-0.49849899999999997 -319,1,-0.6977,-0.8903030000000001,-0.759575,-0.641081,-2.116335,-1.317732,-1.19851,-1.3138459999999998,-1.617385,-1.36466,-0.482055,-0.532817,-0.550673,-0.5050819999999999,-1.47955,-1.322832,-0.946385,-0.82358,-1.2399280000000001,-1.018918,-0.09879099999999999,1.783766,-0.187646,-0.20191099999999998,0.10534500000000001,-0.8209129999999999,-0.72729,-0.09669900000000001,1.6693939999999998,-0.315425 -349,1,-0.716338,-1.295784,-0.71996,-0.675712,-0.134515,-0.418358,-1.156052,-1.019363,0.36116,-0.445866,-0.618382,-1.0075379999999998,-0.607104,-0.648993,1.383436,0.308132,-0.9678540000000001,-0.8008810000000001,2.363553,0.42703800000000003,-0.159426,-0.30272899999999997,-0.203487,-0.301139,-0.41367600000000004,-0.073111,-0.798345,-0.207002,0.823055,-0.25755300000000003 -89,1,0.014663999999999998,-1.211106,0.063706,-0.13533,-0.204652,0.347555,-0.056555999999999995,0.38209299999999996,0.40484000000000003,0.043456,0.145616,-0.9423799999999999,0.156563,-0.008501,1.1984059999999999,0.560187,0.136356,0.560267,1.1112790000000001,0.093901,0.38376,-0.8702559999999999,0.46926,0.053308,-0.5110140000000001,1.041885,0.41334499999999996,0.7192149999999999,0.45134399999999997,0.395687 -419,1,-0.809525,0.528066,-0.83404,-0.7428640000000001,-0.18273399999999998,-0.912638,-1.133983,-1.0894059999999999,0.12172899999999999,-0.704657,-0.8427530000000001,0.49341999999999997,-0.865778,-0.78096,0.38711999999999996,-0.8444969999999999,-1.002443,-0.984276,-0.717843,-0.192455,-0.42831199999999997,0.836678,-0.444567,-0.46967200000000003,-0.044328,-0.896252,-0.8263299999999999,-0.9143979999999999,0.774624,-0.65623 -562,0,0.259021,2.786709,0.638572,0.060502,0.40905,3.4188370000000003,4.307272,1.842324,1.9223189999999999,3.1561630000000003,0.310345,2.6366490000000002,0.470844,0.17636500000000002,0.600616,1.9777580000000001,2.086645,1.170295,1.15509,1.2364899999999999,-0.523235,-0.021506,-0.249525,-0.389146,-0.805359,1.283328,1.382525,0.6948840000000001,0.100217,0.887792 -370,0,0.6441939999999999,0.8716659999999999,0.6564439999999999,0.49998000000000004,0.400283,1.350111,2.094174,1.677876,3.1113869999999997,0.675751,0.6312810000000001,0.930909,0.701509,0.527612,0.075415,0.85962,1.1588319999999999,1.0010860000000001,1.3266850000000001,-0.08755299999999999,0.09394,-0.353531,0.052444000000000005,0.113593,-0.468679,0.765232,0.955794,0.772745,1.383649,0.109729 -546,1,-1.039387,-0.636267,-1.076496,-0.871368,-0.16958299999999998,-1.0550059999999999,-1.095507,-1.3825180000000001,-0.355517,-0.55171,-1.081325,-0.684076,-1.098091,-0.938523,-0.143774,-1.030979,-0.9878170000000001,-1.120082,0.267911,-0.111652,-0.702973,-0.45332,-0.747526,-0.602123,0.015008,-1.01893,-0.721952,-1.0221049999999998,-0.598406,-0.449704 -49,1,-0.231765,1.000313,-0.246067,-0.31955900000000004,-0.708765,-0.529046,-0.211036,0.20698699999999998,-0.048138,-0.818814,-0.181,0.700529,-0.20838299999999998,-0.26703000000000005,-0.629123,-0.5185310000000001,-0.518386,-0.38895100000000005,-0.009559999999999999,-0.796355,-0.618518,0.24701700000000001,-0.559909,-0.44305,-0.862028,-0.651568,-0.362861,0.007115000000000001,-0.500333,-0.6955680000000001 -412,1,-1.305488,0.376621,-1.2108299999999999,-1.018857,-1.0419180000000001,-0.41708500000000004,-0.409653,-1.354653,-0.8489399999999999,0.449654,-1.3434700000000002,0.556251,-1.327108,-1.098077,-1.186348,-0.830283,-0.645945,-1.129548,-1.9628150000000002,0.6297550000000001,-1.034299,-0.063236,-0.837621,-0.737829,-0.50868,0.070525,0.18091400000000002,-1.081636,-0.12498900000000002,0.298098 -417,0,1.4290370000000001,0.321254,1.4844899999999999,1.524843,0.847409,0.92835,0.714407,1.504293,0.165409,1.1556520000000001,0.389869,0.416627,0.45024899999999995,0.42152700000000004,1.113008,0.9998610000000001,0.795994,0.925768,0.9981,0.8282200000000001,3.482267,-0.006991,3.246381,2.9958169999999997,0.385689,0.749583,0.43224399999999996,1.556217,-0.043867,0.764104 -440,1,-0.809525,0.194236,-0.50997,-0.710519,0.295077,0.979241,0.986906,0.6226729999999999,-0.583624,0.63031,-0.896716,-0.486275,-0.83365,-0.805988,-0.513123,0.131884,0.072451,-0.329883,-1.1778620000000002,0.510676,-0.5333399999999999,0.288747,-0.029731,-0.488154,0.5080279999999999,1.168195,1.074496,0.954419,-0.6516810000000001,0.446751 -482,1,-0.298031,-1.198078,-0.366998,-0.387414,0.303844,-0.027768,-0.41924799999999995,0.287688,0.527792,0.516153,-0.18667999999999998,-1.216974,-0.191495,-0.308839,0.764297,0.21148000000000003,-0.388442,0.096492,-0.11908900000000001,0.509259,-0.889208,-1.167627,-0.8732629999999999,-0.6049829999999999,-0.8636950000000001,-0.612445,-0.615186,-0.487462,-0.43131800000000003,-0.381241 -291,1,-0.003974,0.083503,0.054770000000000006,-0.124431,-0.046843,0.31002199999999996,-0.440357,0.5221779999999999,0.09908,0.042901999999999996,0.23650100000000002,-0.044132,0.20846199999999998,0.09218,-0.458325,-0.11581199999999998,-0.369108,-0.018808000000000002,0.246006,-0.606396,-0.42398100000000005,-0.48779300000000003,-0.34407600000000005,-0.340302,-0.5696829999999999,-0.241898,-0.5482090000000001,0.011981,-0.644416,-0.37065 -198,0,1.468383,1.039396,1.761498,1.419368,-0.007390000000000001,1.9455380000000002,0.548412,0.85564,0.472787,0.457966,1.435041,0.744743,1.4635280000000002,1.402458,-0.792091,0.728855,0.28375100000000003,0.48494899999999996,-0.1629,-1.064282,0.11018199999999999,-0.261,0.47866499999999995,0.305448,-0.50568,0.642834,-0.160272,0.067133,-0.7739699999999999,-0.150508 -542,1,0.049868,1.07685,0.0041340000000000005,-0.095249,-1.155891,-0.742153,-0.53295,-0.07775,-0.289188,-0.7972020000000001,0.174018,1.426574,0.112489,0.038995,-0.9685819999999999,-0.610256,-0.599491,-0.48103599999999996,0.103619,-0.850224,-0.368399,0.305076,-0.341105,-0.284418,-0.755357,-0.768936,-0.411933,0.144993,-0.223063,-0.44213900000000006 -320,1,-1.033174,-0.825166,-1.064284,-0.861699,0.34329699999999996,-0.116191,-0.39526100000000003,-0.261383,-0.47361499999999995,0.724517,-1.101206,-0.723636,-1.048251,-0.940514,0.693131,0.128093,-0.270425,-0.23908800000000002,-0.250523,1.416525,-0.135245,0.461109,-0.628223,-0.388486,1.14972,0.959727,0.27010700000000004,0.59107,0.804894,1.43777 -94,0,0.40605,-0.235671,0.48368599999999995,0.253872,0.996451,1.056214,1.190321,1.475363,-0.10799600000000001,-0.088988,0.26490199999999997,0.125743,0.343154,0.144227,0.536567,0.965749,1.019472,1.0119200000000002,0.158383,0.0060090000000000005,0.25851999999999997,-0.45803699999999997,0.415796,0.149676,0.73637,0.652335,0.5562520000000001,1.089053,-0.673475,-0.10398299999999999 -116,1,-1.41959,-1.401633,-1.30823,-1.073352,-0.6342439999999999,-0.422174,-0.5650930000000001,-1.159448,-2.020213,-0.372717,-1.470424,-0.8213729999999999,-1.368711,-1.165198,-0.123848,0.378253,0.048094,-0.6664939999999999,-1.849636,1.252083,-0.331946,-0.431729,0.204913,-0.514776,3.769827,2.354171,2.013834,1.830352,0.399281,0.683158 -429,1,-0.507184,-0.768171,-0.547798,-0.5164449999999999,-1.120822,-1.006469,-1.139405,-1.195078,-1.190291,-1.313123,-0.39969099999999996,-0.37690300000000004,-0.45264,-0.43682299999999996,-1.238299,-1.12043,-0.938852,-0.7885,-1.287391,-1.043018,-0.39619,-0.604636,-0.374767,-0.376165,0.098678,-0.767818,-0.879016,-1.001343,-0.138308,-0.9890909999999999 -518,1,-0.252473,-0.212873,-0.23683400000000002,-0.361925,0.58001,0.266129,-0.708059,-0.076228,-0.515677,0.27620300000000003,-0.354249,-0.248914,-0.30971,-0.46014499999999997,1.810429,1.170425,-0.509095,0.10603499999999999,-0.374655,1.3796680000000001,0.13508499999999998,-0.086823,0.15342999999999998,-0.13128599999999999,-0.58935,-0.122294,-0.591645,0.104441,-0.288445,-0.187576 -323,0,1.870123,1.0068270000000001,1.9014919999999997,1.8588470000000001,1.176179,1.240059,1.257966,2.343279,4.298838,1.0226540000000002,1.7644970000000002,0.516691,1.809525,1.732374,1.4688350000000001,1.575986,2.105477,2.617595,2.765157,0.553204,0.595621,-0.351717,0.567276,0.631953,-0.5186810000000001,-0.065287,0.00020800000000000001,0.307203,0.8642219999999999,-0.170177 -248,1,-0.832304,1.549097,-0.872165,-0.746907,0.768505,-0.728158,-0.766109,-0.810759,0.8222280000000001,-0.137199,-0.9876,1.380033,-0.986877,-0.875668,0.014925,-0.606466,-0.81619,-0.845247,0.31172300000000003,0.069801,-0.561131,0.501025,-0.677726,-0.5213760000000001,0.049342000000000004,-0.845505,-0.6990729999999999,-0.9004479999999999,0.125643,-0.444787 -280,0,1.5429329999999999,1.664716,1.564912,1.482653,2.009061,0.825932,1.454664,1.105356,0.577943,0.734491,1.429361,1.701168,1.40998,1.374017,0.401353,0.776234,1.296937,1.230911,0.329977,-0.084717,0.833467,-0.39163200000000004,0.72024,0.644934,0.11701199999999999,-0.055227,0.264801,0.183924,-0.749754,-0.131973 -391,1,-1.263036,-0.468538,-1.288274,-0.9907299999999999,0.597545,-0.784138,-1.305831,-1.745063,-0.737313,0.26068600000000003,-1.531771,-0.57005,-1.511641,-1.196199,0.536567,-0.5697,-1.114873,-1.2618200000000002,0.6330060000000001,1.1599389999999998,0.40325,1.56423,0.148975,-0.252735,2.926459,-0.325173,-1.057501,-1.9134470000000001,-0.22911700000000002,1.112474 -462,1,-0.17999400000000002,1.026368,-0.20436700000000002,-0.256626,-1.344385,-0.688717,-0.599635,-0.898007,-0.899091,-1.069848,0.077453,1.791923,0.011573,-0.024997,-1.8794990000000003,-0.9875799999999999,-0.678462,-0.813778,-0.381957,-1.200372,-0.626819,-0.55456,-0.563869,-0.43601,-0.561682,-0.485576,-0.403976,-0.880172,-0.775181,-0.719776 -406,1,0.298367,-0.9928950000000001,0.257314,0.11833699999999998,-0.515887,-0.522048,-0.197603,-0.02598,-0.19859200000000002,-0.766169,0.571638,-1.030809,0.507915,0.41271,-0.100363,-0.36635100000000004,-0.424349,-0.093868,-0.27973000000000003,-0.5737909999999999,-0.600833,-1.051691,-0.562879,-0.40718699999999997,-1.027701,-0.727578,-0.450396,-0.4946,-0.671053,-0.822282 -416,1,-1.12222,0.9058639999999999,-1.147684,-0.9161940000000001,0.886862,-0.8585659999999999,-1.011215,-1.1710200000000002,-0.04652,-0.050197000000000006,-1.341198,0.560905,-1.333287,-1.091251,0.57215,-0.810194,-0.857873,-0.9375889999999999,0.779043,0.45538999999999996,0.090331,3.013891,-0.052998,-0.333702,2.5664439999999997,-0.488371,-0.604244,-0.506441,1.198399,0.300746 -411,1,-0.799171,0.12421300000000002,-0.814083,-0.719308,0.198638,-0.674722,-0.7939350000000001,-0.613574,0.15732000000000002,-0.28460599999999997,-0.876835,-0.572377,-0.867014,-0.801153,0.806996,-0.498443,-0.732448,-0.622129,-0.3564,0.085395,-0.752419,0.308704,-0.754456,-0.5895819999999999,-0.627685,-0.901115,-0.707363,-0.699633,-0.407102,-0.604031 -474,1,-0.8965,-1.030349,-0.7887649999999999,-0.786636,0.036445,0.862192,0.30853600000000003,-0.532112,-0.517295,1.332982,-0.922277,-0.853952,-0.888021,-0.841823,0.308838,0.048497000000000005,-0.47268599999999994,-0.8565959999999999,0.180289,0.789944,-0.927466,-1.231674,-0.774752,-0.676334,-0.8556940000000001,0.304702,-0.143362,-0.828103,-1.019759,0.33176300000000003 -96,1,-0.712196,-0.774684,-0.748256,-0.67747,-0.805204,-1.022181,-1.066914,-0.849434,-1.0899889999999999,-0.564455,-0.5530579999999999,-0.337343,-0.584037,-0.579597,0.579267,-0.6400100000000001,-0.802254,-0.503219,0.322675,0.503588,-0.141019,0.533683,-0.22576300000000002,-0.349763,-0.5360149999999999,-0.7650239999999999,-0.682495,0.550517,0.200712,-0.146347 -232,1,-0.809525,2.6222369999999997,-0.8584639999999999,-0.720187,-1.4215360000000001,-1.179499,-1.194624,-1.28796,0.016572999999999997,-0.602692,-0.8257120000000001,3.3789830000000003,-0.8723690000000001,-0.762473,-1.320851,-1.30009,-1.0525120000000001,-1.095861,0.121873,-0.640418,-0.654249,0.780434,-0.681686,-0.547338,-0.8940290000000001,-1.0431860000000002,-0.950636,-1.358853,-0.16736700000000002,-0.47693900000000006 -354,1,-0.859225,-1.6051879999999998,-0.823317,-0.750775,-1.916882,-0.818489,-0.729648,-1.1478760000000001,-0.525383,-0.762844,-0.848433,-1.214647,-0.853833,-0.76873,-1.6809479999999999,-0.828198,-0.549271,-0.88239,-0.44402299999999995,-0.278929,0.061457000000000005,-0.739624,0.229665,-0.25295500000000004,-0.500013,0.469017,0.43622299999999997,0.472657,1.268624,0.391905 -240,1,-0.293889,-1.079202,-0.39171999999999996,-0.34663099999999997,-0.200268,-0.796225,-0.5703699999999999,-0.340866,-0.599801,-1.0443559999999998,-0.138398,-0.858606,-0.189023,-0.22635999999999998,-0.151602,-0.720933,-0.524161,-0.299446,-0.345447,-0.878576,-0.292245,-1.008146,-0.43070600000000003,-0.289258,-0.190334,-0.72646,-0.457359,-0.124275,-0.189161,-0.756845 -378,1,-0.358085,-0.983124,-0.277044,-0.39304,-0.213419,0.357097,-0.07334700000000001,-0.140179,0.786637,0.68905,-0.132717,-0.963324,-0.152364,-0.211286,-0.973563,-0.5469579999999999,-0.581412,-0.62445,-0.071627,-0.542604,-0.95634,-1.224961,-0.873758,-0.6377649999999999,-0.609684,0.24378200000000003,-0.247475,-0.5580229999999999,-0.286024,0.4059 -177,0,0.314933,0.451529,0.48368599999999995,0.176876,0.400283,1.351383,1.506478,1.35355,0.247916,0.623106,0.662522,0.190901,0.713866,0.5062810000000001,0.138753,0.971434,1.136233,1.025075,-0.064325,0.061296,-0.366234,0.121827,-0.190121,-0.192451,-0.137999,0.8641559999999999,0.723363,1.027414,-0.45069,0.298477 -33,0,1.6319780000000002,0.850497,1.6125690000000001,1.6391069999999999,0.812341,2.57468,1.616341,0.972885,1.2477040000000001,1.5712700000000002,1.460602,1.670916,1.480004,1.442275,-0.167259,1.280343,0.9654860000000001,0.696717,0.151081,-0.026595999999999998,0.5436479999999999,-1.1079350000000001,0.327681,0.612372,-0.675354,0.430453,0.102,-0.34926,-0.618989,0.038239999999999996 -251,1,-0.6832050000000001,-0.5239050000000001,-0.719066,-0.653386,-0.6167090000000001,-0.9501700000000001,-0.916991,-0.748634,-0.260067,-1.057102,-0.746188,-0.195392,-0.7698050000000001,-0.7038840000000001,-0.20711100000000002,-0.842033,-0.783673,-0.728142,0.081713,-0.49015200000000003,-0.045014,-0.67848,-0.090125,-0.293659,-0.220335,-0.828738,-0.644696,-0.425174,0.287889,-0.8828020000000001 -224,1,-0.233835,-0.33826300000000004,-0.250833,-0.30198,-0.209036,-0.783502,-0.44899300000000003,-0.271433,-0.638628,-0.42757799999999996,-0.243483,-0.528162,-0.305591,-0.308554,-0.8476,-1.030979,-0.668669,-0.628319,-1.152306,-0.8587290000000001,-0.405935,-0.591754,-0.40694400000000003,-0.344483,-0.33634000000000003,-0.806941,-0.30848400000000004,-0.44431499999999996,-0.692847,-0.49131199999999997 -186,0,1.043864,0.111186,0.951324,0.9306690000000001,-0.393146,-0.06211900000000001,0.391533,0.647036,0.49381800000000003,-0.807177,1.187949,-0.16514,1.096935,1.098139,-0.745834,-0.372605,-0.089257,0.23784299999999997,-0.6959380000000001,-1.2117129999999998,-0.532258,-1.3447069999999999,-0.519316,-0.251195,-1.391716,-0.91084,-0.589324,-0.8235610000000001,-1.1929020000000001,-1.024268 -85,0,1.379337,0.32614,1.338539,1.269946,0.325762,-0.288585,0.208748,0.7551439999999999,1.284913,0.102197,1.230551,-0.179102,1.19991,1.1948379999999998,0.169354,0.018175,0.5612159999999999,1.006761,1.169694,-0.36540300000000003,1.063013,0.46836700000000003,0.948449,0.8858530000000001,-0.190001,-0.502343,-0.127115,0.38993,0.38233,-0.015094 -422,1,-0.751542,-0.978239,-0.7545109999999999,-0.711749,0.400283,-0.237058,-0.201441,-0.062524,-0.184032,-0.536193,-0.714947,-0.7608689999999999,-0.68001,-0.701609,0.8852780000000001,0.236117,-0.223847,-0.101864,0.27156199999999997,0.057042999999999996,-0.575929,-0.876244,-0.593571,-0.537878,-0.385675,-0.30337600000000003,-0.185139,0.201767,-0.2497,-0.685733 -104,1,-0.979333,-0.385488,-0.98416,-0.839901,-0.4589,-0.672177,-0.922652,-1.257355,-0.12093800000000002,-0.466924,-1.033042,8.2e-05,-1.011592,-0.9066690000000001,0.25119400000000003,-0.351758,-0.738851,-0.952034,1.480024,0.285277,-0.181081,0.5754130000000001,-0.279227,-0.378585,0.18468099999999998,-0.183773,-0.102578,-0.516173,0.7939970000000001,-0.130082 -461,0,4.094189,0.927033,4.287337,5.930172,0.146035,1.0899299999999998,1.9723169999999999,2.251919,-0.420228,-0.536193,3.775318,1.624375,3.910226,5.250529,0.8568120000000001,1.790138,3.4488589999999997,3.094784,0.9104770000000001,-0.931027,7.730307000000001,0.161743,7.813534,11.041841999999999,0.203015,1.579542,1.613298,2.30076,-0.432529,0.288642 -142,1,-0.72255,0.176323,-0.732768,-0.663758,0.391516,-0.47751899999999997,-0.9360379999999999,-0.770256,-0.512441,-0.165462,-0.766069,-0.46067700000000006,-0.754153,-0.7306189999999999,0.913744,-0.179679,-0.8598809999999999,-0.781794,-0.6083149999999999,0.40010300000000004,-0.436253,1.253978,-0.459913,-0.417088,-0.125665,-0.45427799999999996,-0.673542,-0.369211,0.275781,-0.101713 -380,1,-0.710125,-0.8381930000000001,-0.665154,-0.711046,1.255083,-0.072298,-0.227827,0.261803,0.715454,0.45464099999999996,-0.811511,-1.472952,-0.774748,-0.763895,1.945643,0.128093,-0.12303,0.169746,0.753487,0.8991,-0.537671,-0.41921,-0.523277,-0.5345770000000001,-0.13533199999999998,-0.430804,-0.36087199999999997,-0.025327000000000002,0.065104,-0.027954000000000003 -123,1,-0.117869,-1.579133,-0.132881,-0.23746399999999998,-0.046843,-0.480063,-0.077665,0.114105,-0.019018,-0.215336,0.10585399999999999,-1.954655,0.095189,-0.040355,0.977793,0.105352,-0.004762,0.228557,0.16203399999999998,0.173287,-0.405213,-0.652898,-0.464368,-0.355263,-1.07437,-0.710811,-0.10191499999999999,0.033069,-0.17463099999999998,-0.373298 -428,1,-0.950341,-0.8772760000000001,-0.9802879999999999,-0.8077310000000001,-1.287398,-1.221866,-1.0861040000000002,-1.129299,-0.837615,-0.726824,-0.851273,-0.6212449999999999,-0.885549,-0.778684,-1.056827,-1.250816,-0.942995,-0.908442,-1.097542,-0.18678499999999998,-0.9516479999999999,-0.453139,-0.9395969999999999,-0.67398,-0.386009,-1.073925,-0.745891,-0.909532,-0.054764,-0.536324 -396,1,-0.304244,0.247975,-0.295809,-0.361046,0.45726999999999995,0.017398,0.343558,0.467362,-0.379783,-0.39266599999999996,-0.17532,-0.093001,-0.159366,-0.275278,0.678898,0.196319,-0.037656,0.126155,-0.020513,-0.2846,-0.6914239999999999,0.20891500000000002,-0.669806,-0.46307200000000004,-0.533015,-0.330203,0.038007,0.30395900000000003,-0.895048,-0.503416 -66,1,-1.213336,0.957974,-1.19832,-0.9666469999999999,0.983301,-0.558944,-0.8542879999999999,-0.752745,-0.036814,0.452425,-1.324157,0.400337,-1.31228,-1.096371,0.57215,-0.504318,-0.8421790000000001,-0.873878,-0.345447,0.8778360000000001,-0.613826,1.440855,-0.597036,-0.57506,1.159721,-0.4431,-0.48918900000000004,-0.399545,0.272148,0.167223 -170,1,-0.573451,-1.634499,-0.604391,-0.5827180000000001,0.268776,-0.812128,-0.709978,-0.315133,-0.11932100000000001,-0.8997209999999999,-0.513297,-1.605595,-0.540376,-0.542624,0.45828500000000005,-0.654413,-0.614306,-0.307442,0.538081,-0.46038199999999996,-0.610577,-1.000163,-0.592086,-0.503995,0.334687,-0.764465,-0.499468,0.099574,-0.15768,-0.585118 -481,1,0.029158999999999997,0.120957,-0.085224,-0.088042,-1.138356,-0.7173430000000001,-0.503205,-0.504095,-0.881295,-0.43866099999999997,-0.064554,-0.011554,-0.133416,-0.147862,-1.170692,-0.96806,-0.738851,-0.727884,-0.8492770000000001,-0.972138,-0.26553699999999997,-0.526982,-0.40100399999999997,-0.262416,-1.112371,-0.8192370000000001,-0.550198,-0.895582,-1.2013770000000001,-0.5968439999999999 -313,1,-0.813667,-2.0855770000000002,-0.775361,-0.725637,-1.015616,-0.583118,-0.906485,-1.0271290000000002,0.7914899999999999,-0.5323140000000001,-0.7348279999999999,-1.994215,-0.751269,-0.699049,-0.739429,-0.846202,-0.9432459999999999,-1.032073,0.078062,-0.25483,-0.988823,-1.5542639999999999,-0.8707879999999999,-0.679833,-0.972366,-0.675042,-0.7249359999999999,-1.156415,-0.026916000000000002,-0.6970810000000001 -53,0,0.896835,-0.251956,0.829202,0.7742140000000001,-0.191501,-0.15626800000000002,-0.047439999999999996,0.272461,0.194529,-0.225865,1.162387,-0.137215,1.166958,1.075386,1.312271,0.836879,1.109868,1.472343,1.023657,0.042867,1.5466469999999999,0.682459,0.9954770000000001,1.286504,-1.047369,0.23092800000000002,-0.12346700000000001,-0.41755,0.750408,0.5034890000000001 -296,1,-1.014537,-1.7680310000000001,-1.037775,-0.858535,-1.7204970000000002,-1.139994,-1.16747,-1.258725,-1.225882,-0.9706530000000001,-0.9137569999999999,-1.614903,-0.940332,-0.8281719999999999,-0.79565,-1.082717,-0.959693,-0.9087,-1.3239,-0.352645,-0.829657,-0.344459,-0.791583,-0.643486,-1.187708,-0.74155,-0.7020569999999999,-0.389812,1.073688,-0.138782 -114,1,-1.3752739999999999,-0.986381,-1.274274,-1.048038,1.7548119999999998,-0.113647,-0.12755899999999998,-0.14627,0.040839999999999994,0.9838629999999999,-1.534044,-0.805083,-1.488162,-1.205869,1.3265040000000001,-0.423205,-0.596101,-0.7655430000000001,-0.593711,1.9183569999999999,-0.861418,-1.143859,-0.748516,-0.690217,0.40669,-0.187126,0.023087,-0.226467,-0.41921,0.004197 -243,1,-0.260756,0.107929,-0.275853,-0.306902,-1.6959490000000002,-0.700167,-0.6538470000000001,-0.815327,-0.384637,-1.149092,-0.107156,1.042608,-0.141243,-0.184551,-1.133686,-0.687389,-0.525166,-0.657209,-0.140994,-1.206042,0.106573,-0.290029,-0.018345,-0.008956,-0.8966959999999999,0.066613,0.060223,-0.279996,0.658389,-0.267766 -140,1,-1.169849,-1.8852790000000001,-1.2132129999999999,-0.9452,-0.393146,-1.159206,-1.305831,-1.745063,0.330422,-0.134983,-1.246621,-1.7033310000000002,-1.265735,-1.0420479999999999,-0.274719,-1.200026,-1.114873,-1.2618200000000002,0.33362800000000004,0.201639,-0.74484,-1.307876,-0.81584,-0.617744,-0.333673,-1.10785,-1.057501,-1.9134470000000001,0.269727,-0.21745799999999998 -152,1,-1.087016,-1.007551,-1.0788790000000001,-0.879102,-0.138898,0.145898,2.635815,0.647036,0.335276,2.324925,-1.248609,-0.91911,-1.1611120000000001,-1.008772,0.771413,1.052926,4.042708999999999,0.764814,2.688487,4.275833,1.513443,2.625622,0.5974729999999999,0.20930100000000001,1.309727,3.93361,12.07268,6.649601,1.806213,9.851593 -268,1,-0.49061800000000005,-0.33174899999999996,-0.535883,-0.49763500000000005,-0.296707,-0.46734,-0.350164,-0.864965,1.1376950000000001,-0.7384609999999999,-0.357089,-0.716655,-0.394974,-0.40582199999999996,-0.150179,-0.798824,-0.625229,-0.845247,0.724279,-0.724057,-0.615991,0.0038950000000000005,-0.65347,-0.486174,-0.50768,-0.20668699999999998,-0.19906500000000002,-0.88666,0.652335,-0.6195390000000001 -91,0,0.033301,0.026507,0.007112,-0.087339,-0.292324,-0.34711,0.05954500000000001,0.502383,-0.557739,-0.868134,0.35294699999999996,0.807574,0.339035,0.208504,-0.31030100000000005,-0.014043,0.293795,0.6683439999999999,-0.345447,-0.259083,-0.333029,-0.681383,-0.391599,-0.239754,0.947045,-0.058021,0.44484399999999996,0.9463090000000001,0.5046189999999999,-0.625213 -255,0,0.025018000000000002,-0.587414,0.024984,-0.095952,0.825491,0.457607,0.23369499999999999,0.347072,0.270565,-0.24248899999999998,-0.047513,-0.521181,-0.022203,-0.149284,0.94221,0.446478,0.114133,0.091333,0.351883,-0.212302,0.071563,-0.738535,-0.150024,-0.10114400000000001,-0.230002,0.07331900000000001,-0.023333,0.263406,0.009408,-0.41603999999999997 -559,1,-0.784675,1.8698990000000002,-0.7440859999999999,-0.714386,-0.112597,-0.016316999999999998,0.43567,-0.275239,-1.2760340000000001,0.18698299999999998,-0.743348,1.079841,-0.718729,-0.7149760000000001,-0.26689,-0.04247,0.28124,-0.202977,-1.546608,0.411444,-0.600472,3.061064,-0.460408,-0.514116,0.386356,0.24266500000000002,0.8450489999999999,0.141749,-0.6855829999999999,0.356727 -14,0,-0.256615,1.031253,0.045834,-0.321493,1.4348100000000001,3.296698,2.02509,1.61697,1.1247530000000001,3.2780769999999997,-0.11283599999999999,0.772668,0.06717999999999999,-0.217827,1.191289,2.368158,1.556825,0.8081470000000001,0.939685,1.9878200000000001,-0.696838,-0.086823,-0.398529,-0.46483199999999997,-0.204001,1.8936419999999998,0.766467,0.727326,-0.112881,1.625761 -100,0,0.149267,1.562124,0.039876999999999996,0.04556,-0.257255,-0.381461,0.21450500000000003,0.057766,-0.40405,-0.552818,-0.146918,1.3241829999999999,-0.161426,-0.20531300000000002,-0.10534500000000001,-0.364456,-0.032007,-0.10392799999999999,-0.739749,-0.579461,0.185253,0.132713,-0.0025039999999999997,0.06166900000000001,-0.389675,-0.592325,-0.179834,-0.304165,-0.713431,-0.544646 -353,0,0.464033,1.228294,0.415178,0.29782,1.4742629999999999,-0.118736,0.627092,0.578516,-0.39919699999999997,0.5782189999999999,0.270583,1.50104,0.248417,0.175512,0.42981899999999995,-0.126046,0.435666,0.42846000000000006,-0.601013,0.261178,0.895907,0.5246109999999999,0.647471,0.506764,1.159721,-0.065287,0.571172,0.918733,0.07963300000000001,0.405522 -543,1,-0.393289,1.871527,-0.440271,-0.44120600000000004,-1.1032879999999998,-0.7389720000000001,-0.796334,-0.53333,-0.6920149999999999,-1.081485,-0.260524,2.0409200000000003,-0.29199899999999995,-0.331307,-0.686767,-0.6741229999999999,-0.7398560000000001,-0.417067,-0.670381,-0.707046,-0.613826,0.689717,-0.656935,-0.49475399999999997,-0.689354,-0.657157,-0.560808,-0.43458199999999997,-0.399837,-0.927436 -165,1,-0.059886,0.02325,-0.147774,-0.173125,-1.221645,-0.9816590000000001,-0.9340229999999999,-0.868924,-0.41213900000000003,-1.279874,0.239341,0.109454,0.14544100000000001,0.100428,-0.8646799999999999,-0.963133,-0.8704280000000001,-0.761674,-1.082938,-1.4371120000000002,-0.7982560000000001,-0.275514,-0.7821779999999999,-0.5213760000000001,-1.1357059999999999,-0.9777950000000001,-0.783358,-0.870115,-0.15768,-0.860485 -43,0,0.23002899999999998,0.37825,0.173913,0.04679,0.9043959999999999,0.7515029999999999,0.451982,0.526746,1.356095,1.0392780000000001,-0.240643,0.230461,-0.191495,-0.311967,0.550801,0.744016,0.12141500000000001,0.326574,0.592845,0.711976,-0.1255,-0.711138,-0.217348,-0.198171,-0.656353,-0.22401300000000002,-0.33302,-0.362723,-0.405891,-0.180768 -201,0,0.85956,0.026507,0.960259,0.630066,0.251241,0.558117,0.377141,1.207374,0.044075,-0.292364,0.969258,0.007063,0.9527690000000001,0.8438760000000001,-0.475405,0.292971,0.18582200000000001,0.6696340000000001,-1.115796,-1.1181510000000001,-0.029133999999999997,-0.705151,0.109867,0.008645,-0.317006,0.011840999999999999,-0.157951,0.268273,-0.557239,-0.43003500000000006 -108,0,2.512079,0.379878,2.964846,2.600686,1.65399,2.833589,3.3036300000000005,2.685877,1.867314,0.772728,2.3126450000000003,0.08851,2.50564,2.429179,2.579016,3.268353,4.238567,3.4404230000000005,2.717694,1.0763,2.9228389999999997,0.595371,3.5562709999999997,2.852806,-0.175333,3.420544,2.39083,2.109352,1.280732,0.469825 -328,0,0.623486,0.765818,0.671337,0.42263199999999995,1.167411,0.257223,0.419839,0.665308,0.327187,-0.107829,0.60856,0.330525,0.6150100000000001,0.45167399999999996,1.461718,0.522284,0.7407520000000001,0.927574,0.497921,-0.003914,0.116678,0.027481000000000002,0.199963,0.089611,-0.11466400000000002,-0.259782,0.019438999999999998,0.344511,-0.6274649999999999,-0.380484 -439,1,-0.281464,-1.0368629999999999,-0.31963800000000003,-0.336962,-1.2698639999999999,-0.970527,-1.005506,-0.494046,-1.237207,-0.933525,-0.030472000000000003,-0.8446440000000001,-0.097993,-0.137624,-1.188483,-0.919734,-0.852851,-0.577763,-0.812768,-0.9834780000000001,-0.689258,-1.019577,-0.623768,-0.46395200000000003,-0.734689,-0.9076540000000001,-0.752025,-0.168071,-1.069402,-0.639587 -162,0,2.166251,0.11607100000000001,2.014678,2.375673,0.501106,0.8291120000000001,1.961283,1.6763540000000001,1.200788,0.459075,1.551487,-0.265204,1.595336,1.590169,1.113008,1.1799,2.0339139999999998,2.055287,0.786345,-0.28034699999999996,1.195471,-0.306358,0.9534,1.2482209999999998,-0.9947,-0.151357,0.278396,0.20014400000000002,-0.44584700000000005,-0.18039000000000002 -325,1,-0.529963,-0.745372,-0.5528609999999999,-0.538243,0.264392,-0.8464799999999999,-0.816483,-0.892069,-0.344192,-0.834885,-0.413892,-0.463005,-0.44151899999999994,-0.469246,0.45828500000000005,-0.524975,-0.713993,-0.71834,-0.381957,-0.41927200000000003,-0.704417,-0.483257,-0.643569,-0.500035,-0.077329,-0.892228,-0.616844,-0.863302,0.0033539999999999998,-0.760627 -183,1,-0.807454,-1.299041,-0.83821,-0.726691,-0.8884920000000001,-0.593296,-0.515199,-0.786396,-1.7629860000000002,-0.536193,-0.771749,-1.0168469999999998,-0.7595069999999999,-0.7192430000000001,-0.410645,-0.43192299999999995,-0.33885,-0.652824,-2.353466,-0.08897000000000001,-0.255431,-0.197498,-0.477239,-0.386506,0.105011,0.659601,0.9037370000000001,0.863582,0.13411900000000002,0.374127 -197,0,0.7228859999999999,-0.159135,0.650487,0.610729,-1.935293,-0.368739,-0.089658,-0.347109,-0.8602639999999999,-1.017757,1.122625,0.593484,1.047507,1.0497889999999999,-1.611917,-0.339629,0.26994,0.228557,-0.151947,-1.332209,0.833828,0.159928,0.715785,0.792566,-0.50368,1.535949,0.9683930000000001,0.42886,0.103849,0.46944600000000003 -498,0,1.342063,-0.45551,1.1657819999999999,1.264672,0.387133,0.347555,0.38913400000000004,0.7871199999999999,-0.632157,0.582098,1.2390709999999998,-0.41180900000000004,1.208148,1.17493,0.344421,0.518494,0.757073,1.106842,0.074411,0.59148,1.397226,-0.311801,0.982606,1.219619,0.31101999999999996,0.372328,0.413013,0.638111,-0.529391,0.552283 -477,1,-0.233835,-0.631382,-0.180538,-0.284225,-1.6889349999999999,-0.341385,-0.641853,-0.797664,-0.358752,-0.38601599999999997,-0.064554,-0.6212449999999999,-0.12353099999999999,-0.157817,-1.998346,-0.9693870000000001,-0.83565,-0.916438,0.005043,-1.054359,-0.901119,-1.162366,-0.729704,-0.578801,-1.244377,-0.689014,-0.729114,-1.12965,-0.8938379999999999,-0.642991 -421,1,0.039513,-1.194821,0.203699,-0.12548499999999999,-0.051226,0.694887,0.238492,-0.057955999999999994,-0.11932100000000001,0.450762,0.15981700000000001,-1.2355909999999999,0.257479,0.0034439999999999996,0.47963500000000003,1.502076,0.705598,0.363201,1.001751,1.5965610000000001,0.508999,0.533683,0.954885,0.2005,0.9783799999999999,1.506886,0.692527,0.649465,0.724982,0.6237729999999999 -374,1,-0.29596,-0.8903030000000001,-0.24130100000000002,-0.369132,-0.9586290000000001,-0.284132,-0.660084,-0.68118,0.683099,-0.384354,-0.124197,-0.7492340000000001,-0.170076,-0.215552,-0.949367,-0.7694489999999999,-0.793968,-0.7379439999999999,0.22044899999999998,-0.8658170000000001,-0.846981,-1.2886440000000001,-0.7396050000000001,-0.579461,-0.937031,-0.536994,-0.669895,-0.885849,-0.134675,-0.611974 -470,1,-1.062166,-0.009318,-1.083645,-0.87084,-0.393146,-0.6365529999999999,-0.693187,-0.7461979999999999,0.44204899999999997,0.07171799999999999,-1.266786,-0.186084,-1.2554370000000001,-1.040342,-0.491062,-0.791432,-0.7447520000000001,-0.8712989999999999,1.556694,0.18888,-0.099513,0.24157399999999998,-0.147053,-0.387386,0.153347,-0.312319,-0.157619,-0.310329,-0.11409200000000001,0.04467 -382,1,-0.766038,0.49386899999999995,-0.592774,-0.6894239999999999,-1.945375,0.427072,0.091208,-0.08231799999999999,-1.1482290000000002,0.528899,-0.5899800000000001,0.798266,-0.544495,-0.588983,-1.9221990000000002,0.056077999999999996,-0.117631,-0.493675,-2.222032,0.537611,-1.031411,0.393979,-0.538623,-0.6773899999999999,-0.666353,1.124602,0.369909,0.790588,0.503408,1.3477459999999999 -510,1,-0.790887,-1.315326,-0.774766,-0.7150890000000001,-1.098904,0.15925699999999998,-0.015297,-0.13713399999999998,-0.48008599999999996,0.8226030000000001,-0.678025,-1.0703690000000001,-0.644999,-0.650984,-1.093833,-0.146514,-0.270425,-0.581116,-1.141353,0.677954,-0.767939,-1.043526,-0.7529710000000001,-0.600582,-0.01966,0.7646729999999999,0.274417,0.565116,0.24914299999999998,1.145003 -132,0,0.662832,0.9775149999999999,0.668358,0.517559,0.312612,0.32592600000000005,0.129588,0.25266700000000003,0.93709,-0.429795,0.577318,0.523672,0.586176,0.440582,0.31595500000000004,0.45595399999999997,0.19461099999999998,0.185997,1.2719209999999999,-0.551109,0.101159,0.087354,-0.010920000000000001,0.07355,-0.721356,-0.332998,-0.321746,-0.41608999999999996,-0.628676,-0.537837 -545,1,-0.190348,0.55575,-0.28836300000000004,-0.26506399999999997,-0.472051,-0.652457,-0.80257,-0.6527069999999999,-0.41861000000000004,-0.798864,-0.14407799999999998,0.9169459999999999,-0.196849,-0.232332,-0.277565,-0.69876,-0.741488,-0.6316729999999999,-0.5389470000000001,-0.6786939999999999,-0.21356399999999998,0.216173,-0.396054,-0.200151,-0.391009,-0.25084,-0.387397,-0.443179,0.039678,-0.458404 -121,0,1.238521,-0.126566,1.135996,1.175019,0.786039,-0.16008499999999998,0.263919,0.80387,-0.010929000000000001,0.034035,1.287353,-0.504892,1.212267,1.2005270000000001,0.643316,0.10724700000000001,0.714386,0.9732290000000001,0.563638,-0.094641,1.1102940000000001,0.660687,1.0043879999999998,1.10301,0.353688,-0.24972199999999997,0.050607,0.6835289999999999,-0.011175,0.29318099999999997 -45,0,1.356558,-0.709547,1.2908819999999999,1.206661,1.5575510000000001,1.62047,2.21795,1.875822,1.453162,0.438017,1.284513,-0.39319299999999996,1.307005,1.197683,0.9635600000000001,1.217803,1.363478,1.3407930000000001,0.34823200000000004,-0.32712800000000003,0.807481,-1.0043360000000001,0.706379,0.686958,-0.24900300000000003,0.808267,0.78404,0.8360059999999999,0.45255500000000004,-0.09830900000000001 -333,1,-0.726692,-0.589042,-0.7500439999999999,-0.6818649999999999,-0.69123,-0.9944459999999999,-1.279358,-1.491235,-0.138734,-0.541181,-0.817192,-1.049425,-0.848066,-0.753372,-0.94652,-1.132559,-1.1026479999999999,-1.185959,-0.140994,-0.281764,-0.6885359999999999,-0.400523,-0.6618850000000001,-0.5559189999999999,-0.474679,-1.0258040000000002,-1.025216,-1.436389,-0.428896,-0.6675770000000001 -134,0,1.294434,0.93029,1.141953,1.247093,0.619463,-0.170263,0.596387,0.354685,0.336894,-0.43478199999999995,1.227711,0.609773,1.162839,1.1948379999999998,-0.146621,-0.137417,0.332715,0.5042939999999999,-0.436721,-0.783596,0.688377,-0.026949,0.44549799999999995,0.616332,-0.346674,-0.629212,-0.11086800000000001,-0.42955299999999996,-0.68074,-0.601383 -182,0,0.795365,1.163157,0.6564439999999999,0.6828029999999999,0.3959,0.638907,0.086411,0.601356,0.867526,0.13101300000000002,0.446671,0.237443,0.380225,0.317717,-0.027775,-0.309875,-0.287124,0.07662999999999999,-0.7068909999999999,-1.03593,-0.127304,-0.121295,-0.22774299999999997,0.014145,-0.805026,-0.159181,-0.409612,-0.329146,-0.614146,-0.5162770000000001 -519,1,-0.376722,-0.641152,-0.406017,-0.450875,0.663299,-0.35856,-0.623143,-0.520083,0.275418,0.090005,-0.39116999999999996,-0.602629,-0.389619,-0.458154,1.14859,0.139464,-0.62774,-0.48928999999999995,1.125883,0.48657700000000004,-0.07858,-0.388004,-0.183686,-0.257796,0.156013,-0.5515260000000001,-0.401986,-0.421281,-0.301764,-0.062753 -207,0,0.731169,-0.102139,0.677294,0.579086,-0.932328,-0.672177,-0.378949,-0.076228,0.605445,-1.067077,0.8187300000000001,0.225807,0.730342,0.70935,-0.614889,-0.5932,-0.242302,0.128476,0.782694,-1.498069,0.651924,-0.655801,0.613809,0.6187520000000001,-0.667687,-0.583942,-0.41226499999999994,-0.091833,0.29031,-0.459161 -99,0,0.012593,0.843983,0.06668400000000001,-0.095249,0.470421,0.30747800000000003,0.226498,0.6379,-0.295659,0.531115,0.083133,0.11178099999999999,0.103427,-0.035236,0.08253200000000001,0.184948,0.063788,0.24429099999999998,0.246006,0.156275,-0.41748500000000005,1.15056,-0.242594,-0.296739,0.321353,0.19404100000000002,0.043644,0.396418,-0.717063,0.248547 -366,0,1.640261,1.3243719999999999,1.570869,1.389484,-0.200268,0.5555720000000001,0.470692,1.531701,0.598974,-0.422591,1.724735,1.75469,1.718907,1.6470509999999998,0.191415,1.185586,0.945398,2.0011189999999996,0.231402,-0.368238,2.0605990000000003,1.224948,2.109791,1.391892,0.466026,1.188316,0.90009,2.20019,2.004782,0.185758 -180,0,3.4895080000000003,1.168042,3.381848,4.105459,0.650148,0.948707,1.256047,2.3478470000000002,-0.072405,-0.17322,3.718515,0.6004649999999999,3.712513,4.536658999999999,0.927977,1.6498970000000002,2.489659,3.58229,-0.042419,-0.722639,1.555309,0.479253,1.462291,1.944135,-0.803359,-0.006044,-0.026649000000000003,0.099574,-0.580244,-0.39637100000000003 -318,1,-1.285815,-0.370832,-1.1509610000000001,-1.0257120000000002,-0.450133,0.766771,0.9053479999999999,-0.0016170000000000002,0.378956,1.194443,-1.444295,-0.090674,-1.313928,-1.167189,0.236249,1.759816,1.3647340000000001,0.004149,1.892581,3.4918980000000004,0.21701399999999998,1.259421,0.446983,-0.355043,0.9347120000000001,2.259159,2.3477259999999998,2.185591,1.7396200000000002,1.544058 -501,0,-0.053674,1.182698,-0.037566,-0.162753,2.061664,0.905449,0.316212,0.5709029999999999,1.213731,1.9037650000000002,-0.087275,1.210157,0.014868000000000001,-0.167771,1.411902,1.208328,0.5888369999999999,0.481596,1.6917790000000001,1.356986,0.25238499999999997,0.564527,0.053434,-0.028318,0.879709,0.731139,0.095037,0.707861,0.45982,1.2104409999999999 -229,0,-0.22141100000000002,0.728364,-0.058415999999999996,-0.306902,1.987143,1.7814139999999998,1.707974,1.265235,0.818992,2.2362599999999997,-0.36844899999999997,0.7075100000000001,-0.276346,-0.43141899999999994,0.8852780000000001,1.4319549999999999,1.013195,0.5079060000000001,1.136836,1.3810850000000001,-0.357572,-0.268257,-0.301503,-0.334582,-0.019327,0.7322569999999999,0.49524300000000004,0.518075,-0.453112,0.68921 -7,0,0.16376300000000002,0.401048,0.099449,0.028858999999999996,1.447961,0.7247859999999999,-0.021054,0.624196,0.47763999999999995,1.726435,-0.11851700000000001,0.35845,-0.072867,-0.21896500000000002,1.604049,1.140102,0.061026,0.28195,1.403355,1.660353,0.643623,0.290561,0.490051,0.23372199999999999,0.5880310000000001,0.268933,-0.23255399999999998,0.435349,-0.688004,0.6116689999999999 -247,1,-0.38914699999999997,-1.299041,-0.067352,-0.42450600000000005,-0.305475,2.1033,2.401216,0.6318090000000001,-0.42346300000000003,1.8760569999999999,-0.351408,-1.205339,-0.289115,-0.40582199999999996,-0.623429,0.573453,0.6101800000000001,-0.235219,-0.787211,0.18320999999999998,-0.731486,-1.409116,-0.234179,-0.5277569999999999,-0.513347,1.701382,1.6474490000000002,0.308825,-0.954377,1.422261 -494,1,-0.36636799999999997,0.453158,-0.35657300000000003,-0.408333,-0.9016430000000001,-0.570395,-0.936517,-1.106307,-0.34581,-0.535085,-0.274725,0.29096500000000003,-0.325775,-0.33045399999999997,-1.637537,-0.977725,-0.888883,-0.937847,-0.360051,-0.555362,-0.29405,0.464738,-0.26734600000000003,-0.3139,0.253684,-0.27710799999999997,-0.612865,-1.010913,0.038467,-0.413771 -174,1,-0.979333,-1.054776,-1.014542,-0.830233,-1.085753,-1.185478,-1.305831,-1.745063,-0.308601,-1.2360950000000002,-0.9847600000000001,-0.963324,-1.0082959999999999,-0.868274,-0.6006560000000001,-1.162123,-1.114873,-1.2618200000000002,0.42490200000000006,-0.43203,-0.268063,1.284821,-0.35199600000000003,-0.403887,0.557363,-0.8555649999999999,-1.057501,-1.9134470000000001,1.465982,-1.0564200000000001 -340,1,0.083001,-0.678606,0.123277,-0.032492,-0.130131,0.526946,0.49563900000000005,0.407978,0.246298,0.204716,0.083133,-0.639862,0.089834,-0.038932999999999995,0.08182,0.18115699999999998,-0.109596,-0.17254,0.366487,0.18746300000000002,-0.202376,-0.809657,-0.09358999999999999,-0.18035,-0.8213600000000001,0.282346,0.215729,0.10281900000000001,-0.21943000000000001,-0.15958599999999998 -125,1,-0.161357,-0.34152,-0.207346,-0.271919,-0.730683,-0.758692,-0.9165120000000001,-0.9678969999999999,-0.868353,-0.671962,-0.07875499999999999,-0.48394799999999993,-0.145362,-0.188249,-0.605638,-0.814553,-0.936592,-0.96751,-0.721494,-0.552527,-0.6737390000000001,-0.6545310000000001,-0.678716,-0.49343400000000004,-0.814026,-0.91151,-0.754877,-1.13257,-0.977382,-0.789374 -209,1,0.23002899999999998,-1.5889030000000002,0.19178499999999998,0.09161699999999999,-0.44574899999999995,-0.22688000000000003,0.115196,-0.16910999999999998,-0.939535,-0.5101479999999999,0.324545,-1.484587,0.255419,0.200825,-1.034766,-0.7967390000000001,-0.375134,-0.447504,-1.6524849999999998,-1.068535,-0.692146,-1.549547,-0.6638649999999999,-0.44745100000000004,-0.916697,-0.733167,-0.44708000000000003,-0.717476,-1.332263,-0.8090430000000001 -347,1,0.20725,-1.261587,0.20667800000000003,0.000381,-0.454517,-0.339476,-0.273883,0.15978499999999998,0.336894,-0.115033,0.179698,-1.058734,0.11949100000000001,0.03928,-0.541589,-0.502991,-0.5363399999999999,-0.351808,-1.061032,-0.521339,-0.22511399999999998,-1.4855,-0.162894,-0.248115,-0.769691,-0.5822649999999999,-0.442438,-0.18267,0.131697,-0.382754 -332,1,-0.888216,0.016737000000000002,-0.9040360000000001,-0.781363,0.43973599999999996,-1.002397,-1.241784,-1.437181,0.632947,-1.037706,-0.8257120000000001,0.132725,-0.825,-0.761051,0.643316,-0.692695,-1.052023,-1.066224,0.468713,-0.35689699999999996,-0.38825,1.35921,-0.44902200000000003,-0.45581099999999997,1.9497529999999998,-0.806941,-0.948182,-1.107752,2.65013,-0.69292 -361,1,-0.428493,0.573662,-0.426569,-0.45597299999999996,-0.805204,-0.557036,-0.724371,-0.890242,-0.426699,-0.962341,-0.234963,0.5306529999999999,-0.27716999999999997,-0.30940700000000004,-0.750104,-0.769638,-0.6950350000000001,-0.6365729999999999,0.012345,-0.827542,-0.5163770000000001,0.584485,-0.414865,-0.425889,-0.514347,-0.28102,-0.46233199999999997,-0.875468,-0.27270500000000003,-0.6955680000000001 -500,1,0.101638,-0.8544780000000001,0.072641,-0.041632999999999996,-0.8271219999999999,-0.23324099999999998,-0.41541000000000006,-0.194995,-1.170878,0.08557200000000001,0.25922199999999995,-0.59332,0.278486,0.098153,0.175759,0.6075659999999999,-0.145504,0.322447,-0.524343,0.8353079999999999,-0.119725,-0.679569,-0.278237,-0.12094500000000001,-0.972699,-0.407331,-0.395687,-0.284862,-1.209853,-0.21102800000000002 -300,0,2.000585,0.091645,1.9014919999999997,2.061007,0.75097,1.0008700000000001,1.630733,1.269803,0.108787,0.85031,1.534446,-0.090674,1.545908,1.598701,1.3265040000000001,1.134417,1.6434540000000002,1.477501,-0.071627,0.385927,2.5474810000000003,-0.101337,2.163749,2.038743,-0.32834,0.366181,0.8118920000000001,0.8976459999999999,-0.206112,0.375262 -283,0,0.47231599999999996,-0.095626,0.584958,0.26442,0.181104,1.376193,1.105405,0.892184,-0.21153400000000003,1.238775,0.600039,-0.120926,0.693271,0.42721499999999996,0.7287140000000001,1.437641,1.3308360000000001,1.0730520000000001,0.235053,0.573051,-0.425425,-0.543493,-0.199032,-0.269457,-0.826026,0.5215529999999999,0.22633899999999998,0.047667,-0.806661,0.108216 -107,1,-0.616938,0.295199,-0.646389,-0.591508,-0.612326,-0.368739,-0.37655,-0.459633,0.133053,-0.6703,-0.5019359999999999,-0.174448,-0.5337850000000001,-0.535229,-0.8248270000000001,-0.685873,-0.783045,-0.766317,-0.765306,-0.303028,-1.029607,-0.585041,-0.9988030000000001,-0.684475,-1.194708,-0.8387979999999999,-0.670558,-1.011886,-0.9725389999999999,-0.9225190000000001 -131,0,0.619345,0.052562,0.525386,0.484159,0.974533,-0.09456200000000001,0.512911,0.560244,-0.10314300000000001,-0.208132,0.378508,0.044295999999999995,0.40081999999999995,0.267377,0.913744,0.34035,0.7256859999999999,0.82414,0.435855,-0.685782,0.24949699999999997,-0.781898,0.11283800000000001,0.17541800000000002,-0.267004,-0.594561,-0.124794,-0.140496,-0.795764,-0.504551 -303,1,-1.078732,-0.18519000000000002,-1.0872190000000002,-0.8880680000000001,0.391516,-0.9533510000000001,-0.9017350000000001,-0.751071,-1.112638,-0.306218,-1.033042,-0.158159,-1.034246,-0.9117879999999999,0.742947,-0.7118359999999999,-0.826485,-0.802687,-1.203419,0.45397200000000004,-0.926383,0.628029,-0.90643,-0.6657069999999999,0.6113649999999999,-0.900724,-0.45802200000000004,-0.421281,-0.318715,-0.180768 -567,0,1.961239,2.237926,2.303601,1.653171,1.4304270000000001,3.9048480000000003,3.197605,2.2899849999999997,1.919083,2.219635,1.838341,2.336457,1.9825240000000002,1.735218,1.525767,3.272144,3.2969440000000003,2.658866,2.137194,1.043695,1.157935,0.686088,1.4385299999999999,1.009503,-0.173,2.017716,1.3022850000000001,0.785721,0.32663400000000004,0.904057 -507,1,-0.9482700000000001,-0.803996,-0.928759,-0.82531,1.4830299999999998,-0.325481,-0.7032619999999999,-0.296404,-0.195356,1.820641,-0.8711540000000001,-0.504892,-0.8534209999999999,-0.820208,1.639632,0.052287,-0.6047640000000001,-0.160932,0.519826,2.404595,-0.8202729999999999,-0.339016,-0.7663369999999999,-0.6168640000000001,1.859749,-0.111675,-0.460674,0.17094700000000002,0.200712,1.7089759999999998 -3,0,-0.281464,0.133984,-0.249939,-0.550021,3.394275,3.893397,1.9895880000000001,2.175786,6.046041000000001,4.93501,-0.7689090000000001,0.253732,-0.592687,-0.764464,3.283553,3.4029089999999997,1.9158970000000002,1.451707,2.8673830000000002,4.910919,0.326373,-0.110409,0.286593,-0.288378,0.689702,2.74428,0.8195180000000001,1.115007,4.73268,2.047511 -516,0,1.157759,0.085131,1.040681,1.076575,0.73782,-0.004231,0.497558,0.554153,0.280271,-0.29458,1.187949,0.300273,1.187553,1.129424,0.742947,0.387729,0.855002,1.1759700000000002,0.17663800000000002,-0.480229,0.504307,-0.534058,0.17422100000000001,0.59455,-0.288338,-0.374915,-0.091637,-0.216734,-0.599617,-0.40469299999999997 -143,1,-0.37051,-0.628125,-0.300575,-0.41624399999999995,-0.051226,0.003403,-0.30314800000000003,-0.204131,1.048717,-0.15327000000000002,-0.348568,-0.78414,-0.338956,-0.40582199999999996,-0.682497,-0.17532,-0.500433,-0.465301,-0.12274000000000002,-0.063453,-0.688897,-0.808569,-0.58268,-0.5213760000000001,-0.57235,-0.550408,-0.55683,-0.683249,0.060261,-0.7216670000000001 -398,1,-0.743259,-0.867505,-0.788467,-0.674833,-0.892875,-0.422174,-0.30842600000000003,-0.899073,-0.502735,0.423054,-0.8711540000000001,-1.03779,-0.8921399999999999,-0.786932,-1.3486049999999998,-1.073809,-0.774382,-1.074916,-1.009919,-0.093223,-0.7928430000000001,-0.959341,-0.793563,-0.601903,-0.927364,-0.6029439999999999,-0.414254,-1.237358,-1.045186,-0.097552 -530,1,-0.573451,0.37499299999999997,-0.5582229999999999,-0.5770930000000001,0.11096600000000001,-0.43807799999999997,-0.650489,-0.20717600000000003,-0.683926,-0.353321,-0.675185,-0.402501,-0.662299,-0.659801,0.77853,-0.136659,-0.451719,-0.116567,-0.77991,0.563128,0.11992699999999999,1.252163,0.14006400000000002,-0.212912,-0.151332,-0.409566,-0.481563,0.221232,-0.26544,-0.5586409999999999 -119,0,0.8926930000000001,0.350566,0.653465,0.66874,-1.1032879999999998,-0.852841,-0.226868,0.059289,3.205219,-1.265465,1.085703,0.167631,0.9156979999999999,0.9303370000000001,-0.878202,-0.703498,-0.199239,0.181612,1.158741,-1.7787540000000002,0.52488,-0.005177,0.243031,0.30148800000000003,-1.0057,-0.95326,-0.297873,-0.31357399999999996,3.5836449999999997,-0.715993 -259,0,0.459891,3.885905,0.567086,0.271451,2.451803,1.922,1.430197,1.321574,0.988859,2.020138,0.398389,3.320807,0.48320100000000005,0.256,0.707364,1.128731,1.083503,0.904617,1.020006,0.5248520000000001,-0.589283,0.11094100000000001,-0.47674399999999995,-0.38100500000000004,-0.565349,0.004575,-0.099594,-0.25566500000000003,-1.2828629999999999,-0.164881 -450,1,-0.720479,0.407562,-0.70745,-0.656375,-1.6569349999999998,0.544758,0.238012,-0.412736,-0.963802,0.8630559999999999,-0.641103,0.523672,-0.62358,-0.633919,-2.151351,0.039021,-0.012922999999999999,-0.646375,-1.6889939999999999,0.470983,-0.538393,0.6117,-0.45100200000000007,-0.44217,-0.062328999999999996,1.964621,1.151752,0.605669,0.36295700000000003,1.940466 -192,1,-1.304866,-0.7893399999999999,-1.340697,-1.0139340000000001,-2.6826950000000003,-1.443878,-1.305831,-1.745063,-1.604443,-1.017203,-1.251733,-0.248914,-1.286742,-1.0431860000000002,-1.9115240000000002,-1.533193,-1.114873,-1.2618200000000002,-0.579108,0.23707899999999998,-0.185052,6.655278999999999,-0.314869,-0.410268,-1.7760650000000002,-1.04749,-1.057501,-1.9134470000000001,2.112542,-0.7969390000000001 -514,0,0.271446,0.38802,0.19476300000000002,0.15191300000000002,-0.340543,-0.280951,0.06914,-0.039684,-1.001011,-0.7983100000000001,0.26206199999999996,-0.051114,0.21793600000000002,0.133704,-0.299627,-0.348157,-0.175008,-0.14365,-0.9149940000000001,-0.517087,-0.06919600000000001,-0.034207,-0.11685699999999999,-0.040639,-0.6963550000000001,-0.512962,-0.073732,-0.384946,-1.092406,-0.785592 -241,1,-0.635576,-0.8642479999999999,-0.6973229999999999,-0.592739,-1.256713,-1.122819,-1.006321,-1.128081,0.000395,-0.893071,-0.48489499999999996,-0.988922,-0.550261,-0.507357,-1.216949,-1.334392,-0.982669,-0.976022,-0.969759,-0.745321,-1.046209,-0.984016,-1.044049,-0.689337,-1.2587110000000001,-1.148426,-0.8422120000000001,-1.303215,-0.404681,-0.921006 -402,1,-0.442989,-0.173791,-0.326191,-0.454742,-1.7130450000000002,-0.14290899999999998,-0.536308,-0.738889,0.495436,-0.635942,-0.331527,-0.232625,-0.320832,-0.368849,-1.62615,-0.48043900000000006,-0.605518,-0.776119,0.227751,-0.539768,-0.61166,0.149042,-0.232199,-0.44283,-1.1373719999999998,0.651217,0.08707999999999999,-0.185915,0.699555,-0.034006 -148,1,-0.08680700000000001,-0.9489270000000001,0.039876999999999996,-0.199845,-0.033692,0.12236099999999998,0.182841,0.68967,-0.339339,-0.394329,0.088814,-0.956343,0.08242000000000001,-0.042062,0.237673,-0.04247,-0.049332,0.165104,-0.319891,-0.281764,-0.593975,-0.866265,-0.369322,-0.421048,-0.445011,-0.140179,-0.024991,0.508343,-0.726749,-0.47920799999999997 -395,1,-0.27939400000000003,-0.054915,-0.322915,-0.344697,-1.129589,-0.834393,-0.89996,-0.5404869999999999,-0.6111260000000001,-0.9894950000000001,-0.019112,-0.490929,-0.091402,-0.13022899999999998,-1.132262,-0.961427,-0.778274,-0.423257,-0.622919,-0.7311449999999999,-0.9195260000000001,0.8493790000000001,-0.806434,-0.6087229999999999,-0.556682,-0.712488,-0.682163,-0.428418,-0.405891,-0.8892329999999999 -299,1,-1.105653,-0.2373,-1.106878,-0.910393,-0.792053,-1.06951,-1.10635,-1.269232,-1.0899889999999999,-0.8963959999999999,-1.027362,0.884367,-1.034658,-0.9120729999999999,0.36577,-0.689284,-0.8016260000000001,-0.7781819999999999,-0.42576800000000004,0.391598,-0.42723,-0.133996,-0.28566199999999997,-0.43513,1.043049,-0.617475,-0.44044799999999995,0.114173,1.7069290000000001,-0.689894 -149,1,-0.192419,-0.5239050000000001,-0.29998,-0.271919,-1.545592,-0.457162,-0.5554979999999999,-0.828574,-0.8910020000000001,-0.765061,-0.10999600000000001,-0.32105300000000003,-0.15854200000000002,-0.198772,-1.2041389999999998,-0.76907,-0.7531640000000001,-0.919018,-1.236277,-0.991984,-0.560049,-0.833607,-0.640104,-0.415108,-1.40105,-0.5342,-0.46730600000000005,-0.96809,-0.8780969999999999,-0.6880029999999999 -60,1,-1.087016,-1.339752,-1.114026,-0.900022,-0.213419,-0.989865,-1.20182,-1.3523690000000002,1.061659,-0.207578,-1.1239270000000001,-1.026155,-1.1293950000000001,-0.9754959999999999,1.212639,-0.449737,-0.978777,-0.929077,3.4004209999999997,0.96431,0.399279,0.406679,0.220754,-0.12578599999999998,0.15768,-0.809735,-0.8033520000000001,-0.584464,2.577483,0.816303 -61,1,-1.388321,0.22191999999999998,-1.3463559999999999,-1.0664959999999999,1.382207,-0.537316,-0.874006,-1.322068,0.115258,-0.378258,-1.570397,0.393356,-1.536767,-1.23175,1.9883419999999998,-0.27879499999999996,-0.738224,-1.022993,0.059807000000000006,0.6765359999999999,-0.169532,1.5424579999999999,-0.184676,-0.48287299999999994,1.62974,0.34326599999999996,-0.06279,-0.41154799999999997,1.577375,-0.282517 -126,0,0.12855899999999998,1.622376,0.176892,-0.056048,0.645764,0.217146,0.515309,0.278552,0.9209120000000001,-0.274077,-0.146918,1.2566979999999999,-0.173371,-0.234039,-0.269025,-0.487451,-0.45134300000000005,-0.466075,-0.184806,-0.212302,-0.628624,-0.384375,-0.551493,-0.451191,-0.984366,-0.767818,-0.462001,-0.797445,-0.8853620000000001,-0.881289 -372,0,1.3296379999999999,-0.624868,1.335561,1.150408,-0.577257,0.189156,0.624693,1.248486,-0.27624499999999996,0.150409,2.057032,-0.974959,2.031952,2.079354,0.266139,0.893733,1.310748,1.975325,0.589194,-0.137169,-0.230167,0.167186,-0.227248,-0.028098,-0.8716950000000001,0.07108400000000001,0.082106,0.26502800000000004,-0.45916599999999996,0.216396 -82,0,2.843411,1.293432,3.1107970000000003,2.955784,1.09289,2.247704,1.801046,2.620403,-0.882913,1.172276,3.150487,1.3078940000000001,3.275896,3.478653,0.707364,3.073153,3.077232,3.4971699999999997,0.063458,0.711976,1.7761930000000001,0.466552,2.235529,1.75272,0.37502199999999997,1.75783,0.842397,1.379409,-1.197745,0.793608 -425,1,-1.068378,0.531323,-1.112239,-0.8864860000000001,-0.866574,-1.166203,-1.246581,-1.3523690000000002,-0.8926200000000001,-0.184857,-1.163689,0.463168,-1.185414,-0.9885790000000001,-1.081023,-1.236034,-1.083862,-1.1287479999999999,-0.663079,0.22573800000000002,-0.7942859999999999,0.22524499999999997,-0.8326709999999999,-0.632265,-0.439011,-1.105559,-0.988733,-1.341334,-0.737646,-0.523463 -337,0,1.7127400000000002,1.415565,1.603633,1.744582,0.764121,1.453165,0.917342,1.3733440000000001,1.2590290000000002,0.817615,1.3185950000000002,0.49807399999999996,1.274053,1.243188,-0.37008,0.679581,0.215954,0.30903400000000003,0.516175,-0.278929,0.855484,0.568156,0.744002,1.054166,0.169014,0.754054,0.240265,1.0322799999999999,0.137751,0.2459 -386,1,-0.650071,-1.0401200000000002,-0.584136,-0.61647,-1.304933,-0.071025,0.169888,-0.353352,-0.36198800000000003,0.23796599999999998,-0.544538,-1.209993,-0.543259,-0.548596,-1.087428,-0.494842,-0.256238,-0.6082,-0.604664,-0.178279,-0.5001359999999999,-0.7002520000000001,-0.380708,-0.44833100000000004,-0.8786950000000001,0.267256,0.38284,-0.150228,-0.161313,0.31285 -184,0,0.317004,0.383135,0.19476300000000002,0.16263699999999998,-0.099446,0.48114399999999996,0.43567,0.121718,0.44366700000000003,0.763308,0.327386,0.7261270000000001,0.286312,0.158448,-0.41206800000000005,0.016280000000000003,-0.440043,-0.420162,-0.308938,0.05279,-0.721019,-1.308602,-0.753466,-0.457791,-1.250377,-0.644302,-0.46929499999999996,-0.93873,-1.068191,-0.46143 -58,1,-0.422281,-0.5581020000000001,-0.506991,-0.450875,-1.326851,-1.223647,-1.296979,-1.575895,-0.747019,-1.166825,-0.30596599999999996,0.004736,-0.3855,-0.363161,-1.1215870000000001,-1.259344,-1.106185,-1.154336,0.026949,-1.103975,-0.004229999999999999,-0.005177,-0.134183,-0.16230799999999998,0.150013,-0.9437030000000001,-1.034556,-1.2375200000000002,0.164388,-0.304456 -88,1,-0.5051140000000001,0.785359,-0.47065200000000007,-0.537716,-0.08629500000000001,-0.050669,-0.138113,0.089742,0.115258,0.480133,-0.5019359999999999,0.584176,-0.502069,-0.536936,-0.614889,-0.187449,-0.359691,-0.295835,0.432204,0.176122,-0.38752800000000004,0.517354,-0.328235,-0.42654899999999996,0.023675,-0.030635000000000003,-0.16126600000000002,0.183924,-0.116514,0.252708 -117,0,0.526157,0.275658,0.590915,0.37692600000000004,2.429885,1.232425,0.950924,1.3413680000000001,1.106957,1.249858,0.21094000000000002,-0.60961,0.274778,0.078528,1.411902,1.147683,1.006917,1.039777,1.260968,0.69213,0.077338,-0.48616000000000004,0.060859,0.018546,-0.01866,0.008487,-0.059142999999999994,0.14823699999999998,-0.547553,0.033701 -459,1,-1.159494,1.830816,-1.168535,-0.932895,-0.9367110000000001,-0.9120020000000001,-0.960889,-1.004137,-0.937918,-0.655891,-1.241793,2.073499,-1.247611,-1.0352219999999999,-1.175673,-1.1007209999999998,-0.921401,-0.9927879999999999,-0.6959380000000001,-0.464635,-0.819551,0.853008,-0.8034640000000001,-0.639305,-0.150999,-0.714164,-0.576724,-0.92332,-0.581455,-0.577553 -221,1,-0.266969,-1.3918620000000002,-0.183517,-0.341005,0.229323,0.098824,-0.069509,-0.36096500000000004,0.26571100000000003,-0.120575,-0.161119,-1.254207,-0.139183,-0.266177,0.621966,0.2816,-0.128052,-0.113729,0.549034,0.032944,-0.535145,-1.304066,-0.42328000000000005,-0.424789,-0.39667600000000003,-0.130678,-0.213986,-0.557375,-0.256965,-0.33169 -223,0,0.681469,0.751162,0.555172,0.36462100000000003,1.0008350000000001,1.232425,0.601665,0.506951,1.7670119999999998,1.2443170000000001,0.460872,0.22348,0.437892,0.302644,0.436936,0.304342,0.325182,0.404988,0.45045799999999997,0.032944,-0.20887199999999997,-0.536961,-0.307938,-0.17925,-0.758357,-0.09714400000000001,-0.26703699999999997,-0.512443,-0.34171999999999997,-0.251123 -128,1,-0.032964999999999994,-1.19645,-0.040545,-0.207404,0.27315900000000004,0.21650999999999998,-0.365516,0.42168199999999995,-0.502735,-0.34113000000000004,0.27626300000000004,-0.674768,0.31349699999999997,0.055775,1.3265040000000001,1.447116,0.31388299999999997,0.9394389999999999,0.691421,0.265431,0.09285800000000001,-0.270071,-0.034681000000000003,-0.010937,0.655034,0.9150159999999999,0.00485,1.7492470000000002,0.361746,0.44977700000000004 -294,1,-0.573451,-1.334867,-0.5576270000000001,-0.5746319999999999,-0.112597,-0.681083,-1.054776,-0.7792399999999999,-0.8602639999999999,-0.8160430000000001,-0.39969099999999996,-1.282132,-0.419688,-0.46298900000000004,0.022040999999999998,-0.386818,-0.9531649999999999,-0.7655430000000001,-0.633872,-0.25483,-0.810167,-0.950269,-0.755446,-0.593102,-0.325673,-0.764465,-0.839858,-0.6193390000000001,-0.8236120000000001,-0.9089020000000001 -541,1,-0.010186,0.985657,0.185828,-0.12601300000000001,0.071514,1.0555780000000001,0.632369,0.089742,0.46308,1.017112,0.097334,1.32651,0.15821,0.004297,-0.568632,0.353616,0.151924,-0.258434,0.22044899999999998,0.086813,-0.54489,-0.250114,-0.12428199999999999,-0.379025,0.032342,1.176579,0.21208200000000002,-0.028571,0.016673,0.876066 -111,1,-0.6086550000000001,-0.033745,-0.543926,-0.620865,-0.16081600000000001,-0.186167,-0.24269899999999997,-0.062524,-1.091607,0.050660000000000004,-0.425252,0.342161,-0.404448,-0.49626499999999996,0.21134099999999997,0.313818,0.222232,0.291236,-0.27973000000000003,1.1202459999999999,-0.22655799999999998,1.0634709999999998,-0.07675900000000001,-0.43688999999999995,1.95642,0.835094,0.633839,1.809264,0.10869200000000001,0.792851 -266,1,-0.908925,-0.44573999999999997,-0.8632299999999999,-0.801227,-0.485202,-0.01759,-0.386625,-0.538203,0.063489,-0.447528,-1.001801,-0.079038,-0.934566,-0.877375,0.036986,0.196319,-0.312987,-0.580343,0.402996,0.299453,0.163598,-0.036021,0.27916799999999997,-0.291238,0.143013,0.577443,0.054586,0.300714,1.7553599999999998,-0.18039000000000002 -568,1,-1.410893,0.7641899999999999,-1.432735,-1.075813,-1.859019,-1.207552,-1.305831,-1.745063,-0.048138,-0.751207,-1.808401,1.221792,-1.814389,-1.347789,-3.112085,-1.150752,-1.114873,-1.2618200000000002,-0.8200700000000001,-0.561032,-0.070279,0.383092,-0.157449,-0.46615200000000007,0.049342000000000004,-1.163516,-1.057501,-1.9134470000000001,0.75283,-0.382754 -362,1,-0.52168,0.050934,-0.579073,-0.528926,-0.112597,-0.44761999999999996,-0.703741,-0.47942799999999997,-0.253596,-0.640929,-0.38833,-0.10463599999999999,-0.41598100000000005,-0.45019,0.028446,-0.470394,-0.777395,-0.802429,-0.192108,-0.137169,-0.663633,0.12364100000000001,-0.6589149999999999,-0.507735,-0.477679,-0.50402,-0.55053,-0.291837,-0.175842,-0.628617 -271,1,-0.817808,-1.546564,-0.8635280000000001,-0.743743,0.150419,-0.658818,-0.694146,-0.412736,-0.271392,-0.206469,-0.8058310000000001,-1.454335,-0.813055,-0.7590600000000001,0.140888,-0.535587,-0.7049529999999999,-0.551195,-0.15559800000000001,-0.013837,-0.775157,-1.247458,-0.842571,-0.597722,-0.189667,-0.7968810000000001,-0.632428,-0.478054,-0.438583,-0.36989299999999997 -187,1,-0.674921,-0.698148,-0.680345,-0.6312369999999999,-0.003007,-0.955896,-0.5761270000000001,-0.07166,-0.531855,-0.719066,-0.686545,-0.48860200000000004,-0.7121390000000001,-0.667195,0.098188,-0.813606,-0.6366539999999999,-0.42635200000000006,-1.079287,-0.261918,-0.577734,-0.81891,-0.5564439999999999,-0.49453400000000003,-0.045328,-0.937499,-0.40165500000000004,0.008737,-0.190372,-0.803369 -155,1,-0.554813,-0.074456,-0.615412,-0.556174,-0.467667,-0.480063,-0.373672,-0.494807,0.34336500000000003,-0.145512,-0.5331779999999999,-0.31407199999999996,-0.5642659999999999,-0.553431,-0.698865,-0.711647,-0.627112,-0.660562,0.578241,-0.073377,-0.668325,-0.42556099999999997,-0.6841619999999999,-0.5242359999999999,-0.50768,-0.550967,-0.39635,-0.6280979999999999,-0.309029,-0.49585100000000004 -5,0,-0.165498,-0.313836,-0.11500899999999999,-0.24431999999999998,2.0485130000000003,1.721616,1.2632430000000001,0.9058879999999999,1.7540689999999999,2.241802,-0.476375,-0.835335,-0.387148,-0.50565,2.237421,1.244335,0.866302,0.824656,1.0054020000000001,1.890005,-0.25506999999999996,-0.592662,-0.321304,-0.289258,0.156347,0.44554399999999994,0.160025,-0.069124,0.13411900000000002,0.486846 -511,1,-0.136507,-1.318583,-0.16564500000000001,-0.211623,-0.809587,-0.9743440000000001,-0.777623,-0.533787,-0.9168860000000001,-1.2482870000000001,0.193899,-1.068042,0.11084100000000001,0.073409,-0.8283860000000001,-1.02681,-0.685995,-0.606394,-0.5572020000000001,-1.320868,-0.674821,-1.077091,-0.588621,-0.431609,-0.110997,-0.7549640000000001,-0.566114,-0.20051300000000002,-0.57419,-0.762518 -252,0,1.8659810000000001,-0.014204,1.564912,1.850057,1.693442,2.170731,2.766787,2.072245,-0.245507,2.535505,1.591249,0.12341600000000001,1.595336,1.567416,0.700248,1.526713,1.919664,1.250514,-0.287032,0.59148,1.302665,-0.792602,0.618264,1.1544940000000001,0.48036,1.4023729999999999,1.197177,1.2837049999999999,-0.711009,1.3102989999999999 -480,1,-0.606584,0.35708,-0.5489890000000001,-0.585707,-0.50712,-0.167719,-0.528632,-0.878669,-0.8004060000000001,-0.368838,-0.558739,-0.293129,-0.563442,-0.567652,-0.390718,-0.49199899999999996,-0.7487699999999999,-0.8679450000000001,-1.269136,0.0060090000000000005,-0.6704899999999999,-0.048721,-0.588126,-0.529737,-0.710022,-0.492842,-0.5942970000000001,-1.076283,-0.726749,-0.732636 -513,1,0.101638,-1.373949,0.036898,-0.032668,-0.441366,-0.391004,-0.11028800000000001,-0.346348,-0.44449399999999994,-0.746219,0.128576,-1.310057,0.095601,0.011123000000000001,0.139464,-0.287323,-0.082603,-0.140039,-0.265127,-0.9069280000000001,0.040885000000000005,-1.076184,-0.151014,-0.071001,-0.696021,-0.411243,-0.051185,-0.510658,-0.623833,-0.5726359999999999 -86,0,-0.012256999999999999,0.581805,0.03392,-0.12601300000000001,-0.077528,-0.360469,0.30086,0.12019500000000001,0.192911,-0.858159,0.100174,0.505055,0.093953,-0.019024,-0.136658,-0.092312,0.396746,0.011887,0.96159,-0.9125979999999999,0.054961,1.820053,0.21530900000000003,-0.032277999999999994,0.776038,0.24322399999999997,0.723363,0.9430649999999999,0.44044700000000003,-0.20648899999999998 -105,0,0.008451,-0.533675,-0.025651999999999998,-0.093843,2.3597479999999997,0.9900559999999999,1.7530700000000001,1.278939,0.39836900000000003,3.133996,-0.288925,-0.867914,-0.196026,-0.35462899999999997,3.091407,1.36752,1.485262,1.214661,0.41394899999999996,2.001996,-0.051872,-0.5317,-0.22576300000000002,-0.124905,0.040342,0.203542,0.7571829999999999,0.338023,-0.614146,1.249401 -242,1,-0.7639670000000001,0.371736,-0.598731,-0.7166710000000001,0.102199,1.466524,2.2616080000000003,0.109537,0.658832,2.535505,-0.802991,-0.255896,-0.743031,-0.755078,-0.031333,0.533655,0.8286370000000001,-0.525659,0.884921,1.969391,-0.5860350000000001,0.771362,-0.24606,-0.526877,-0.125998,1.881346,1.886843,0.21798800000000002,-0.071715,1.8459029999999998 -312,1,-0.43056400000000006,-1.510738,-0.45337700000000003,-0.460192,-0.5684899999999999,-0.21288400000000002,-0.45714899999999997,-0.464354,-0.544797,-0.078459,-0.38833,-1.377542,-0.398681,-0.42886,-0.599233,-0.47115200000000007,-0.6061449999999999,-0.604589,-0.768957,-0.198126,-0.28394400000000003,-1.011412,-0.257445,-0.333482,-0.182334,0.12306099999999999,-0.017365000000000002,-0.179426,-0.391362,0.225852 -473,1,-0.583805,2.01483,-0.660686,-0.565491,-1.672278,-1.2858610000000001,-1.305831,-1.745063,-0.795553,-0.9152379999999999,-0.527497,2.485389,-0.599278,-0.538926,-1.378495,-1.333445,-1.114873,-1.2618200000000002,-0.403862,-0.453294,0.145552,4.409122,0.008881,-0.11456500000000001,0.099344,-0.9632639999999999,-1.057501,-1.9134470000000001,1.3158450000000002,-0.249231 -364,1,-0.318739,-0.647666,-0.402145,-0.38161300000000004,-0.485202,-0.551311,-0.6514479999999999,-0.68118,-0.25845,-0.450299,-0.20656100000000002,-0.544452,-0.267285,-0.29149,-1.209121,-0.89794,-0.8410489999999999,-0.8818739999999999,-0.59006,-0.8204540000000001,-0.890652,-1.096686,-0.905935,-0.596622,-0.8823620000000001,-0.725342,-0.576392,-1.02389,-0.924107,-0.650934 diff --git a/examples/data/breast_homo_host.csv b/examples/data/breast_homo_host.csv deleted file mode 100644 index 503643565..000000000 --- a/examples/data/breast_homo_host.csv +++ /dev/null @@ -1,229 +0,0 @@ -id,y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15,x16,x17,x18,x19,x20,x21,x22,x23,x24,x25,x26,x27,x28,x29 -565,0,1.53672,2.047399,1.42194,1.494959,-0.69123,-0.39482,0.236573,0.733827,-0.531855,-0.973978,1.704854,2.085134,1.615931,1.7238419999999999,0.102458,-0.017833,0.6930430000000001,1.2636690000000002,-0.217664,-1.058611,1.300499,2.260938,1.156857,1.291565,-0.42401000000000005,-0.069758,0.252202,0.808431,-0.189161,-0.49055600000000005 -447,1,0.033301,-0.47830900000000004,-0.040545,-0.0898,-0.42821499999999996,-0.42090200000000005,-0.31754099999999996,-0.480037,1.131224,-0.614884,0.19105899999999998,-0.37923,0.16109300000000001,0.056629,-0.325246,-0.292629,-0.604011,-0.678876,0.297119,-0.558197,-0.6668810000000001,-1.0790870000000001,-0.6851520000000001,-0.45295100000000005,-0.7483569999999999,-0.7694949999999999,-0.4746,-0.794687,0.24187899999999998,-0.689894 -488,1,-0.610726,-0.665579,-0.616305,-0.581488,0.886862,-0.6779029999999999,-0.591,-0.25057199999999996,-0.15653,-0.20536100000000002,-0.695066,-0.725963,-0.678775,-0.6666270000000001,1.16994,-0.22194,-0.577646,-0.453952,0.151081,0.171869,-0.122251,-0.11403800000000001,-0.154479,-0.280898,0.652367,-0.701869,-0.44376400000000005,-0.020461,0.118379,-0.220106 -151,1,-1.486271,0.658341,-1.464904,-1.108862,1.342755,1.1242809999999999,1.275717,-0.545359,0.681481,3.582864,-1.6780389999999998,0.328198,-1.594021,-1.282659,-0.164412,0.495752,0.543639,-0.7026060000000001,1.4982790000000001,2.808612,-0.763969,1.351952,-0.8034640000000001,-0.6628470000000001,1.796413,1.603016,1.5131629999999998,-0.25566500000000003,0.30847199999999997,3.0203729999999998 -263,0,0.339783,0.975886,0.257314,0.189884,-1.050685,-0.467976,-0.22159,-0.44044799999999995,-0.352281,-0.8675799999999999,0.42111000000000004,0.021025,0.330797,0.294964,-1.278151,-0.913101,-0.586434,-0.527465,-0.966108,-1.1861959999999998,-0.632955,-0.395624,-0.65941,-0.399487,-1.40505,-0.9150870000000001,-0.622812,-1.074175,-1.2607059999999999,-0.9266790000000001 -563,0,1.66097,0.6078600000000001,2.139779,1.6496549999999999,0.365215,1.0454,1.860055,2.125538,0.045693,0.8192780000000001,1.9292259999999999,1.349781,2.1019759999999996,1.968434,0.9635600000000001,2.260135,2.870075,2.540213,1.23176,0.849484,2.010431,-0.34627399999999997,2.916691,1.726318,-0.214002,0.984878,1.543668,2.342934,0.0033539999999999998,0.9146479999999999 -194,0,-0.039178,0.34242399999999995,0.337735,-0.168554,-0.033692,1.3392959999999998,0.895753,0.8845709999999999,0.160555,0.16980399999999998,0.2081,0.912292,0.347273,0.046959,0.57215,1.774977,1.015706,1.02817,-0.272428,0.55604,-0.453216,-0.462029,0.35886799999999996,-0.333042,0.346688,1.438701,0.783046,1.1409610000000001,0.594217,0.19559200000000002 -433,0,1.323425,0.855382,1.133017,1.269946,0.290694,0.585471,0.5709609999999999,0.85564,0.17188,-0.044655,1.3327959999999999,0.623736,1.307005,1.294382,0.38711999999999996,0.654944,0.88639,0.993606,0.479666,-0.20946700000000001,1.493953,1.295707,0.805385,1.398493,0.344354,0.8608030000000001,0.706785,1.0614780000000001,0.39807,0.272377 -244,0,1.114272,0.790245,1.121103,0.942974,0.610696,0.270582,0.353153,0.6363770000000001,0.031132999999999997,-0.432566,1.497524,0.979777,1.529432,1.422367,0.451169,0.975224,1.457641,1.030234,0.607449,-0.39659,0.429958,1.061657,0.5796520000000001,0.44163900000000006,1.189722,0.39356599999999997,0.24059699999999998,0.615401,0.159545,0.058287 -489,0,0.6027779999999999,0.143755,0.5968720000000001,0.357589,-1.3794540000000002,0.24004699999999998,-0.11748399999999999,-0.41471499999999994,2.873574,-0.42757799999999996,0.727846,0.21184499999999998,0.6232479999999999,0.576531,-1.522249,-0.629587,-0.656742,-0.666752,0.125524,-1.353473,-0.569793,-1.177425,-0.540108,-0.38254499999999997,-1.4580520000000001,-0.615799,-0.585676,-1.054223,-0.112881,-0.7924 -465,1,-0.171711,-0.028860000000000004,0.230506,-0.25855900000000004,-0.537805,1.974164,1.8394259999999998,0.321186,-0.090201,2.26951,-0.25200300000000003,0.195555,-0.21003000000000002,-0.31850900000000004,-0.9621770000000001,0.34035,0.153179,-0.531076,-0.768957,0.215815,-0.44816300000000003,-0.731822,0.248971,-0.363624,-0.704022,2.296605,1.489953,0.305581,-0.639573,1.6408909999999999 -483,1,-0.27111,-0.349662,-0.341978,-0.341181,-0.5465720000000001,-0.7612369999999999,-0.470102,-0.36294499999999996,-0.619215,-0.794985,-0.12135699999999999,-0.383884,-0.173371,-0.23830500000000002,0.223439,-0.469447,-0.5438729999999999,-0.44673,-0.29068299999999997,-0.271841,-0.584952,-0.491058,-0.644559,-0.43336899999999995,-1.2653780000000001,-0.96645,-0.49449399999999993,-0.41576599999999997,-0.931372,-0.858972 -397,1,-0.523751,-0.7518859999999999,-0.49269399999999997,-0.509062,-1.623181,-0.464796,-0.393821,-0.481864,-1.476638,-0.743448,-0.37696999999999997,-0.425771,-0.367377,-0.416914,-1.132974,-0.291682,-0.18706,-0.20865100000000003,-0.867532,-0.750991,-0.14895999999999998,0.087354,-0.09804500000000001,-0.214892,-0.5400149999999999,0.5193180000000001,0.44948599999999994,0.331534,-0.22427399999999997,0.102921 -52,1,-0.656284,-0.707918,-0.702684,-0.621217,-0.787669,-1.050935,-0.8643149999999999,-0.786396,-0.18726700000000002,-0.546722,-0.621222,-0.24425999999999998,-0.669713,-0.617993,-0.9785450000000001,-1.0770309999999998,-0.867289,-0.913859,0.20584499999999997,-0.240654,-0.6419779999999999,-1.059492,-0.666341,-0.503115,0.056343,-0.955607,-0.622812,-0.6157699999999999,-0.070504,-0.43873500000000004 -147,1,-0.003974,-0.033745,-0.004802000000000001,-0.124606,-1.432057,-0.013772999999999999,-0.10645,-0.465267,-0.078876,0.456304,0.233661,-0.120926,0.241826,0.098437,-1.066078,0.23422199999999999,0.021352,-0.343038,-0.24687199999999998,0.302288,0.060736,1.255792,0.200458,-0.019957,-0.41701000000000005,1.301771,0.700816,0.5634939999999999,1.576164,2.108031 -27,0,1.043864,0.257745,0.972174,0.9183629999999999,0.062747,-0.270773,0.347396,0.5237,-0.9055620000000001,-0.539518,1.273153,0.22348,1.241101,1.248876,-0.139504,0.042812,0.755818,0.732313,-0.418466,-0.8232889999999999,1.615944,1.146931,1.369225,1.170555,1.2363899999999999,0.097352,0.627208,1.186379,0.289099,0.159658 -436,1,-0.376722,-0.211245,-0.36104,-0.445953,-0.480818,-0.566578,-0.9639110000000001,-0.772997,0.666921,-0.36551300000000003,-0.357089,0.058258000000000004,-0.383029,-0.41435500000000003,-0.35584699999999997,-0.483471,-0.88926,-0.722725,0.180289,0.095318,-0.139576,-0.812016,-0.133193,-0.30443899999999996,-0.377008,-0.6627460000000001,-0.823213,-0.8587600000000001,0.204344,-0.535946 -434,1,0.008451,-0.836565,-0.147774,-0.18121099999999998,-0.46328400000000003,-0.631464,-0.720533,-0.531351,-0.6078899999999999,-0.8686879999999999,0.2081,-0.546779,0.120315,0.0535,-0.506718,-0.636788,-0.6947840000000001,-0.519727,-0.871183,-0.817619,-0.36948200000000003,-0.995265,-0.620798,-0.36120399999999997,-0.42834399999999995,-0.49340100000000003,-0.5730770000000001,-0.5701890000000001,-0.6093029999999999,-0.615379 -190,0,-0.10958599999999999,1.873156,-0.025651999999999998,-0.207756,0.9175469999999999,4.315794,2.766307,0.95309,3.664668,3.3612010000000003,0.026330000000000003,0.891349,0.098896,-0.12795399999999998,0.792763,2.595576,1.372267,0.445226,2.089732,1.789355,-0.430117,1.620475,-0.373282,-0.18959,0.309686,6.143482,2.808608,0.788966,3.7035129999999996,2.997678 -407,1,-0.387077,0.217034,-0.46558900000000003,-0.412728,-1.6810450000000001,-0.385914,-0.424046,-0.8922209999999999,-0.667748,-0.134983,-0.362769,0.484112,-0.384677,-0.399281,-1.483819,-0.401411,-0.34575500000000003,-0.780246,-0.845627,-0.234983,0.33972800000000003,1.0544,-0.155469,0.019865999999999998,-0.34334000000000003,1.07989,0.658375,0.261784,0.744354,1.488834 -202,0,1.832848,1.1403590000000001,2.077228,1.943226,0.9306979999999999,1.033313,2.480375,2.416366,0.480876,0.20360799999999998,2.6023389999999997,1.717457,2.7569,2.929741,1.2624549999999999,1.972072,3.308243,2.9168060000000002,1.417958,-0.07195900000000001,0.53679,0.622586,0.8915209999999999,0.942177,0.762038,1.438142,1.9127049999999999,2.085021,0.146227,0.5295880000000001 -455,1,-0.252473,2.594554,-0.314872,-0.307605,-0.664929,-0.713526,-0.96986,-0.563022,-1.14014,-0.398762,-0.21224099999999999,2.65992,-0.231861,-0.27783800000000003,-0.278277,-0.570079,-0.760948,-0.419904,-1.59407,-0.373909,-0.232332,1.283007,-0.286652,-0.250975,-0.400009,-0.727578,-0.794367,-0.43279799999999996,-0.5935630000000001,-0.306347 -452,1,-0.658355,1.9871459999999999,-0.660984,-0.627369,-0.50712,-0.436806,-0.43699899999999997,-0.661538,-0.7340770000000001,-0.111154,-0.6041810000000001,2.08048,-0.626051,-0.604056,-0.853294,-0.755046,-0.605769,-0.760127,-0.717843,-0.24915900000000002,-0.77227,0.885666,-0.668321,-0.582541,0.097678,0.023019,-0.082353,-0.426634,-0.374411,0.19180999999999998 -385,0,-0.099232,0.9824,-0.150752,-0.215139,-0.051226,-0.611744,-0.022493,0.324232,-0.6855439999999999,-0.863701,0.134256,0.930909,0.08242000000000001,0.027902999999999997,-0.678938,-0.719796,-0.061511,0.09778099999999999,-0.674032,-1.2244709999999999,0.037998000000000004,0.744147,0.023732,-0.161208,0.423691,-0.450366,0.066191,0.642977,-0.380465,-0.349468 -188,1,-0.766038,0.130727,-0.8248059999999999,-0.6874899999999999,0.141652,-0.981341,-0.8835049999999999,-1.0894059999999999,0.484112,-1.007782,-0.658144,-0.442061,-0.687837,-0.642736,0.308838,-0.923335,-0.8194540000000001,-0.861239,-0.341796,-0.7084630000000001,-0.791399,1.2866360000000001,-0.9183110000000001,-0.56912,0.263351,-0.933475,-0.5415770000000001,-0.901259,1.313423,-0.6834640000000001 -159,1,-0.809525,-1.217619,-0.8694850000000001,-0.721769,-0.669312,-1.089867,-1.216885,-1.143155,-0.263303,-0.393221,-0.9165969999999999,-1.472952,-0.958868,-0.8193549999999999,-1.509439,-1.2728,-1.076078,-1.0918889999999999,-1.349457,-0.760915,-0.445276,-0.826349,-0.523772,-0.47957299999999997,-0.299672,-1.081135,-0.996658,-1.333386,-0.5051760000000001,-0.427388 -32,0,0.9548180000000001,1.044281,0.8589870000000001,0.814646,1.3602889999999999,0.64654,1.375025,1.06729,1.0179790000000002,0.048444,0.82157,1.0914760000000001,0.858032,0.69513,1.660981,0.857725,1.919664,1.8411959999999998,1.593204,0.14493399999999998,0.7064229999999999,0.328662,0.56084,0.603791,0.409024,0.298554,0.614276,-0.109676,0.057839,0.022353 -264,0,1.099776,0.5948319999999999,0.9900450000000001,0.976374,1.0271370000000002,0.01549,0.559926,1.275894,0.509996,-0.456949,0.8698530000000001,0.6470060000000001,0.808603,0.777609,0.064029,-0.27273000000000003,0.022733,0.421754,0.202194,-0.991984,0.0546,-0.86826,-0.023296,0.11183299999999999,-0.849361,-0.749934,-0.378445,-0.311627,-1.102093,-0.8396819999999999 -245,1,-0.991758,0.616002,-1.000245,-0.839901,0.838642,-0.964802,-0.739243,-0.719399,-0.028725,-0.358309,-1.0358829999999999,0.132725,-1.0400129999999999,-0.902118,0.75718,-0.845823,-0.508342,-0.469944,-0.272428,0.22715500000000002,-0.12008599999999998,2.5312759999999996,-0.172795,-0.376605,2.999795,-0.649332,-0.439122,-0.075612,1.7214580000000002,-0.088853 -535,0,1.66304,-0.032117,1.576826,1.632076,-0.244104,0.376817,0.8209120000000001,1.5256100000000001,0.285125,-0.457503,1.82414,0.365431,1.8877869999999999,1.8575139999999999,0.586383,1.3182459999999998,1.502839,2.148145,1.15144,-0.040772,1.059043,-0.411409,0.910827,1.043825,-0.821026,0.038109,0.270438,0.391552,-0.12862200000000001,-0.41831 -430,0,0.016734,0.308227,0.540279,-0.084174,0.417818,2.89275,3.021056,2.02352,-0.056227,1.7486009999999998,0.21946,0.7540520000000001,0.41729700000000003,0.085638,0.22130500000000003,2.239288,2.316401,1.243034,0.837458,0.876418,-0.549221,-0.620421,0.296989,-0.355263,-0.025327000000000002,2.0484560000000003,1.570525,1.710316,-0.6722640000000001,0.752378 -443,1,-1.103582,-0.385488,-1.129217,-0.904065,-1.5092079999999999,-1.201318,-1.114601,-1.406728,-0.326396,-0.919117,-1.0103209999999998,-0.225644,-1.035894,-0.8930170000000001,-1.063232,-1.131801,-0.8646520000000001,-0.975248,2.045921,-0.725474,-0.806197,2.404272,-0.786633,-0.5988220000000001,1.2263899999999999,-0.680072,-0.396681,-0.11129800000000001,-0.40831300000000004,0.263678 -154,1,-0.310456,-0.8430790000000001,-0.285682,-0.357354,0.676449,-0.18235,0.137744,-0.264733,1.534051,0.13212100000000002,-0.277565,-0.91911,-0.274287,-0.329885,-0.179357,-0.366919,0.051861000000000004,-0.363415,0.037902,-0.103146,-0.484255,-0.76956,-0.5183260000000001,-0.38606599999999996,0.514361,-0.296669,-0.047206,-0.366616,0.865433,-0.11949100000000001 -9,0,-0.24419000000000002,2.443109,-0.28627800000000003,-0.29740900000000003,2.320295,5.112877,3.9954330000000002,1.620015,2.370444,6.846856,-0.47353500000000004,1.1054389999999998,-0.329482,-0.509063,1.582699,2.563358,1.7388720000000002,0.94176,0.797298,2.783096,-0.38825,0.693345,-0.40942,-0.360764,0.036008,2.609587,1.5098479999999999,0.409395,-0.32113600000000003,2.3773459999999997 -63,1,-1.2961690000000001,-1.04989,-1.241212,-1.00286,-1.490797,-0.550038,-0.635617,-0.9704860000000001,0.6167699999999999,0.052877,-1.407089,-1.263516,-1.3497629999999998,-1.1205450000000001,-1.362838,-0.318972,-0.363081,-0.699511,1.932741,0.968562,0.016703,1.9016979999999999,-0.127747,-0.370004,0.565697,0.7769689999999999,0.37223,0.617023,2.58838,0.766752 -329,0,0.302508,-0.076084,0.19178499999999998,0.166328,0.448503,-0.271409,0.29846100000000003,-0.150838,-0.26653899999999997,-0.244706,0.605719,0.602792,0.639724,0.488932,1.433252,0.454059,1.143766,0.796798,0.20949600000000002,0.357575,0.597064,0.435708,0.046999,0.382454,1.173055,0.67525,0.8785379999999999,0.010359,2.382547,0.9048139999999999 -536,0,-0.202773,1.3992799999999999,-0.088202,-0.2677,0.246858,0.12108900000000002,0.725441,0.3288,-0.328014,-0.024151,0.040531,0.758706,0.07418200000000001,-0.07135599999999999,0.529451,0.209585,0.721919,0.321673,0.4176,-0.42210699999999995,-0.7307640000000001,1.15056,-0.480704,-0.47957299999999997,-0.309339,0.019665000000000002,0.48264300000000004,0.15634800000000001,-0.730382,-0.014715 -65,0,0.215534,1.255978,0.218592,0.07825700000000001,1.4216600000000001,0.5555720000000001,0.14494,0.7125100000000001,0.679863,0.286177,0.185378,1.082168,0.223703,0.038142,1.483068,0.825508,0.475842,1.0671190000000002,0.516175,0.530523,-0.171337,0.11638399999999999,-0.20596199999999998,-0.11214400000000001,-0.11266400000000001,-0.132913,-0.289916,0.007115000000000001,-0.18673900000000002,-0.21594499999999997 -438,1,-0.132365,0.379878,-0.189474,-0.231136,-0.9016430000000001,-0.8916459999999999,-1.077804,-0.848216,-0.627304,-0.822139,-0.07875499999999999,0.07222100000000001,-0.13547599999999999,-0.17715699999999998,-0.677515,-0.777787,-0.946385,-0.6703640000000001,-0.9369,-0.8601469999999999,-0.22836199999999998,0.836678,-0.264871,-0.23557399999999998,-0.40167600000000003,-0.811971,-0.864792,-0.771654,-0.04871,-0.555615 -269,1,-0.9482700000000001,-0.076084,-0.9159510000000001,-0.826541,0.049596,0.004675,-0.090138,-0.435575,-0.478468,0.16980399999999998,-0.9705590000000001,0.25605900000000004,-0.9255040000000001,-0.881641,0.8425790000000001,0.46543,-0.054229,-0.522306,-0.524343,0.825384,-0.308126,0.493767,-0.314869,-0.43116899999999997,0.620366,1.250353,1.375893,0.917111,0.8157909999999999,1.443065 -36,0,-0.078524,0.7625609999999999,0.26624899999999996,-0.142361,0.536175,1.0784790000000002,1.182165,0.458226,1.1166639999999999,0.9672379999999999,0.034851,0.565559,0.06841599999999999,-0.062255,0.13305899999999998,0.10345599999999999,0.5411279999999999,0.18212799999999998,0.267911,-0.21939,-0.430117,-0.35897399999999996,-0.10349100000000001,-0.339422,-0.387675,0.24993,0.53901,-0.030193,-0.031758999999999996,0.085899 -98,1,-0.664567,-1.3869770000000001,-0.7238319999999999,-0.647058,0.470421,-0.439986,-0.383747,-0.45856800000000003,-0.208298,0.200283,-0.717787,-1.500877,-0.726143,-0.689095,-0.46473000000000003,-0.5513170000000001,-0.588066,-0.397721,-0.699589,0.42845500000000003,-0.626819,-1.229678,-0.688617,-0.5409579999999999,-0.296005,-0.680631,-0.49615200000000004,-0.796796,-0.488225,-0.470508 -392,0,1.021085,0.6078600000000001,1.0377020000000001,0.841015,1.566318,0.871734,1.358234,1.4844979999999999,0.46308,0.9949459999999999,0.38702800000000004,0.158322,0.429654,0.255431,1.397669,0.9828049999999999,1.259272,1.088786,0.42855299999999996,0.658107,0.872808,0.20710100000000004,0.895481,0.58465,0.07601000000000001,0.212484,0.591066,0.745169,-0.24485700000000002,0.165332 -69,1,-0.581734,-0.963583,-0.643112,-0.572523,-0.12136400000000001,-1.168303,-0.807368,-0.849434,-0.837615,-1.099772,-0.38265,-0.651497,-0.436576,-0.43341,0.138753,-0.9854959999999999,-0.6562399999999999,-0.52308,-0.809117,-0.8884989999999999,-0.60769,-0.623505,-0.690597,-0.48419300000000004,0.307019,-1.110309,-0.5319619999999999,-0.508387,0.24187899999999998,-0.71448 -517,0,1.5450030000000001,-0.072828,1.5857620000000001,1.345536,0.40905,0.487505,0.701933,0.710987,-0.569064,0.41086300000000003,1.636691,0.225807,1.587098,1.590169,0.522334,0.505228,0.656634,1.170811,-0.035117,-0.130081,0.370767,-0.622598,0.39005500000000004,0.42601700000000003,-0.650686,-0.136825,-0.045548000000000005,-0.002618,-1.207431,-0.152777 -92,1,0.018805000000000002,-0.541818,-0.082245,-0.087866,-1.3926049999999999,-0.82994,-0.658165,-0.22088000000000002,-1.413545,-1.21282,-0.243483,-1.05408,-0.297765,-0.29348,-1.623304,-1.019419,-0.705455,-0.578795,-1.5539100000000001,-1.3633959999999998,0.001905,-0.115852,-0.081709,-0.087723,-0.853361,-0.843828,-0.607228,-0.158339,-1.1929020000000001,-0.8926370000000001 -552,1,-0.49683,1.681,-0.570733,-0.502558,-0.393146,-0.940628,-0.8907010000000001,-0.7556390000000001,-0.798788,-1.058764,-0.38549,2.359728,-0.4374,-0.41805200000000003,-0.9678700000000001,-1.17501,-0.86415,-0.8751680000000001,-0.995315,-0.9111809999999999,-0.592892,0.272418,-0.687627,-0.474733,0.5980310000000001,-0.734843,-0.617175,-0.404087,-0.190372,-0.782566 -505,1,-1.1739899999999999,-1.243674,-1.125643,-0.971217,2.990984,0.712699,0.091688,-0.108204,-0.085347,2.906791,-1.26423,-1.431064,-1.1471069999999999,-1.0875540000000001,2.073741,2.19949,0.376658,0.553561,0.8958729999999999,4.671344,-0.471984,0.31414699999999995,-0.5341670000000001,-0.49871400000000005,4.909873,1.307919,0.663017,0.438593,0.699555,2.906898 -566,0,0.561361,1.374854,0.579001,0.42790600000000006,-0.809587,0.35073499999999996,0.32676700000000003,0.41406899999999996,-1.104549,-0.318409,0.702284,2.0455740000000002,0.672676,0.5779529999999999,-0.840484,-0.03868,0.046588,0.105777,-0.809117,-0.8955870000000001,0.184892,-0.257371,0.27669299999999997,0.180698,-0.379342,0.661277,0.510827,0.612157,-0.891416,0.036726999999999996 -163,1,-0.556884,0.48898400000000003,-0.592774,-0.575863,0.562476,-0.130186,-0.49600900000000003,-0.497395,-1.02366,0.380938,-0.507616,0.681912,-0.49918500000000005,-0.5414859999999999,0.344421,-0.053841,-0.440671,-0.533913,-0.9515040000000001,0.6822060000000001,-0.397995,0.796763,-0.45100200000000007,-0.41334799999999994,1.433065,0.350531,-0.021344,-0.072368,-0.212166,0.587461 -358,1,-1.302174,-1.299041,-1.250743,-1.017099,-1.3531520000000001,-0.8235790000000001,-0.8528959999999999,-1.0199719999999999,-0.755108,-0.533977,-1.490873,-0.8842040000000001,-1.451091,-1.177143,-0.955772,-0.5185310000000001,-0.522153,-0.6476649999999999,0.432204,0.483742,0.47976499999999994,-0.030577999999999998,0.698459,-0.223473,1.2963930000000001,0.196276,0.008166,0.5294300000000001,0.947766,0.143394 -442,1,-0.20691500000000002,-1.33161,-0.278832,-0.305847,-1.1032879999999998,-0.936175,-1.137102,-1.240758,-1.685332,-0.878109,-0.098636,-0.814392,-0.14824500000000002,-0.196212,-0.582865,-0.704256,-0.982418,-1.0055049999999999,-1.484542,-0.612066,-0.17639000000000002,-1.330918,-0.312394,-0.24195500000000003,-0.203001,-0.777878,-0.800832,-0.995827,-1.001598,-0.465591 -15,0,0.246596,1.865014,0.5015569999999999,0.110075,1.553167,2.56641,2.064909,0.8617309999999999,2.1310119999999997,2.7793349999999997,0.11721500000000001,1.9199119999999998,0.196105,0.011123000000000001,1.248222,1.045345,0.942887,0.637649,1.7940060000000002,1.130169,-0.126943,-0.333573,0.006406,-0.171329,-0.478012,0.9457549999999999,0.514474,-0.145362,-0.23880300000000002,0.6320939999999999 -503,0,3.007006,-0.294295,3.1048400000000003,3.342525,-0.5465720000000001,0.688526,0.51435,1.7022389999999998,0.01172,-0.619317,2.5455360000000002,0.125743,2.476807,2.921209,-0.20924600000000002,0.43889799999999995,0.98934,1.325317,-1.1194469999999999,-1.128074,3.197139,-0.855742,3.350833,3.077223,-0.42934399999999995,0.451691,0.260823,1.583793,0.177707,-0.339633 -176,1,-1.037316,-0.209616,-1.018414,-0.862051,-0.099446,0.259131,0.366586,-0.236107,-0.463908,1.7873919999999999,-1.199475,-0.286147,-1.1273360000000001,-1.002515,0.044814,0.474905,0.526062,-0.303315,-0.5206930000000001,2.6030599999999997,0.093579,1.894441,0.131649,-0.282878,1.9397520000000001,3.498789,2.911395,2.07691,2.210615,5.342837 -327,1,-0.662496,-0.5581020000000001,-0.7303850000000001,-0.627897,-1.36192,-1.147374,-1.268736,-1.3193270000000001,-1.180585,-0.752315,-0.595661,-0.316399,-0.654061,-0.594102,-1.389881,-1.239824,-1.0954629999999999,-1.11758,-1.5685129999999998,-0.297358,-0.6196010000000001,-0.5572819999999999,-0.693072,-0.514116,-0.770691,-1.03905,-1.018243,-1.272557,-0.71222,-0.771596 -1,0,1.805927,-0.369203,1.535126,1.8904889999999999,-0.375612,-0.43044399999999994,-0.146749,1.087084,-0.24389,0.28119,1.829821,-0.353632,1.6859549999999999,1.908708,-0.8269620000000001,-0.487072,-0.023846,0.548144,0.001392,-0.8686520000000001,0.499255,-0.876244,0.263327,0.742402,-0.605351,-0.692926,-0.44078,0.260162,-0.80545,-0.099444 -404,1,-0.639717,-1.437458,-0.689578,-0.610845,-1.208494,-1.188468,-1.0697450000000002,-1.015252,-0.9751270000000001,-1.341385,-0.507616,-1.009865,-0.563442,-0.528403,-0.678938,-1.1111440000000001,-0.850089,-0.732011,-0.8784850000000001,-0.810531,-0.078941,-0.560729,-0.130717,-0.224133,0.220349,-0.9494030000000001,-0.624138,-0.24268800000000001,1.108801,-0.8937719999999999 -24,0,2.1103389999999997,0.957974,2.077228,2.345788,2.109883,0.6586270000000001,0.946607,1.444909,1.152255,0.648043,0.716485,0.486439,0.742699,0.710203,1.120124,0.7838149999999999,0.79976,1.103489,0.669515,0.071219,1.44956,-0.571797,1.2816049999999999,1.369891,-0.33100599999999997,-0.37212,-0.148667,-0.080478,-0.7097979999999999,-0.375945 -139,1,-0.900641,-1.61333,-0.9153549999999999,-0.785054,0.189871,-0.458434,-0.889933,-0.43390100000000004,-1.2922120000000001,-0.892517,-0.808671,-1.3728879999999999,-0.781338,-0.768161,1.426135,0.175472,-0.53295,-0.02474,-0.148296,-0.294523,-0.240994,0.228873,-0.5024850000000001,-0.30818,1.4097309999999998,0.531055,-0.332357,1.2739719999999999,-0.57419,-0.133486 -101,1,-1.7269009999999998,-0.999409,-1.6933610000000001,-1.222423,1.14111,-0.852841,-1.305831,-1.745063,0.050546,0.547186,-2.029648,-1.36358,-1.984504,-1.454443,1.4688350000000001,-0.543168,-1.114873,-1.2618200000000002,0.432204,2.180614,-0.6535270000000001,0.5282399999999999,-0.650005,-0.671142,1.049716,-0.8181189999999999,-1.057501,-1.9134470000000001,0.732247,0.11540299999999999 -350,1,-0.619009,-0.9668399999999999,-0.704471,-0.594321,-1.437317,-1.205517,-1.159698,-1.0961049999999999,-0.274627,-0.8697969999999999,-0.700746,-0.5165270000000001,-0.752505,-0.665205,-1.476703,-1.289477,-1.010591,-0.962094,-0.5133909999999999,-0.777926,-0.186856,-0.987826,-0.317344,-0.31478,-0.152666,-1.0332379999999999,-0.86025,-0.892175,0.19586900000000002,-0.426253 -90,1,-0.032964999999999994,0.559006,-0.129902,-0.135154,-0.9147930000000001,-0.494058,-0.864986,-0.687423,-0.612744,-0.636496,0.13993599999999998,1.1007850000000001,0.107134,0.022215000000000002,-0.471135,-0.346452,-0.725418,-0.499092,-0.46227700000000005,-0.586549,-0.11936400000000001,-0.192055,-0.290612,-0.144707,-0.724356,-0.40789000000000003,-0.68581,-0.517471,0.037256,-0.341146 -191,1,-0.52168,-0.354547,-0.542734,-0.529278,-1.687182,-1.046355,-1.057031,-1.018297,-1.167642,-0.844305,-0.38549,0.49341999999999997,-0.409803,-0.41947399999999996,-0.631257,-0.726429,-0.724162,-0.52308,-0.42941899999999994,0.010262,1.176342,0.963683,1.1147799999999999,0.29290700000000003,-0.8233600000000001,-0.42353900000000005,-0.33600399999999997,0.9365760000000001,1.596747,0.786799 -383,1,-0.43263500000000005,-0.414799,-0.35836,-0.49236199999999997,0.45288599999999996,0.668169,0.23225500000000002,-0.252247,-0.13226300000000002,1.543562,-0.49341599999999997,-0.421117,-0.466645,-0.546037,0.557917,0.480591,-0.375134,-0.518953,-0.11908900000000001,0.436961,-0.521069,-0.623868,-0.370807,-0.465052,-0.10866400000000001,0.646746,0.519779,-0.20862399999999998,-0.261808,1.168455 -11,0,0.85956,0.261002,0.8709020000000001,0.73554,0.31699499999999997,1.950627,0.596387,1.010951,1.441838,1.1556520000000001,0.469393,-0.325708,0.47908199999999995,0.358672,0.052641999999999994,0.47111499999999995,0.134849,0.44213100000000005,0.110921,-0.28034699999999996,0.363187,-0.420843,0.345502,0.304128,-0.423343,0.8457129999999999,-0.132088,0.16608,-0.055974,0.132046 -314,1,-1.515262,-0.527162,-1.5074969999999999,-1.125913,0.102199,-1.123391,-1.305831,-1.745063,0.39028,-0.154378,-1.570681,-0.160486,-1.560245,-1.2334559999999999,0.785646,-0.869323,-1.114873,-1.2618200000000002,1.282873,1.529934,-0.24676900000000002,2.8306419999999997,-0.318829,-0.495634,4.569858999999999,-0.639831,-1.057501,-1.9134470000000001,4.954254,1.144247 -352,0,3.491579,-0.34152,3.6350279999999997,4.137101,0.9043959999999999,2.159281,1.789052,2.451387,1.276824,0.23297800000000002,3.2953339999999995,-0.425771,3.38711,3.8540739999999998,1.319387,2.500819,3.113641,3.672569,0.5271279999999999,-0.22505999999999998,2.128091,-0.6639649999999999,2.156324,2.480977,-0.224002,0.9474319999999999,0.356978,0.532674,0.339952,-0.155047 -235,1,-0.19449,0.749534,-0.267811,-0.290202,-0.16081600000000001,-0.6550020000000001,-1.006897,-0.532569,-1.091607,-0.430903,-0.027632,0.456187,-0.089755,-0.14644000000000001,-0.402816,-0.6612359999999999,-0.931319,-0.772766,-1.075636,-0.630495,-0.527927,0.5191680000000001,-0.593571,-0.401907,0.11601199999999999,-0.6510090000000001,-0.815388,-0.284862,-0.958009,-0.329042 -305,1,-0.7929579999999999,0.967744,-0.7705960000000001,-0.7103430000000001,-1.618359,-0.751695,-0.9585379999999999,-1.011902,0.555294,-0.9141290000000001,-0.717787,1.210157,-0.7306739999999999,-0.676012,-1.538617,-0.8994559999999999,-0.867038,-0.9231450000000001,0.45045799999999997,-0.569538,-0.555717,1.032628,-0.448032,-0.486834,-0.306339,-0.117823,-0.528315,-0.778304,1.380017,-0.599114 -375,1,0.145126,-1.064546,0.173913,-0.033546,-0.388763,0.004675,-0.291634,0.15978499999999998,0.408076,0.31333099999999997,0.580158,-0.7492340000000001,0.590295,0.38000300000000004,0.173624,0.747807,-0.279842,0.13028199999999998,0.6512600000000001,0.414279,-0.8325440000000001,-1.320576,-0.750991,-0.559439,-0.843694,-0.411243,-0.41060699999999994,0.02658,-0.145573,-0.037411 -215,0,-0.10751500000000001,0.204007,-0.085224,-0.22937800000000003,0.597545,1.16245,0.918301,0.773416,1.179757,1.216609,-0.075915,-0.5491060000000001,-0.041562,-0.216121,0.44405200000000006,0.897523,0.128195,0.183159,1.07477,0.901935,-0.537311,-0.041464,-0.46189300000000005,-0.388266,-0.36034099999999997,0.497521,0.238607,0.414261,-0.139519,0.28939899999999996 -466,1,-0.304244,-0.035373,-0.189474,-0.336611,0.119734,0.6401789999999999,0.854974,0.053198,-0.546415,-0.12223699999999998,-0.280405,0.337507,-0.246689,-0.335573,-0.68392,0.0864,0.247342,-0.356451,-0.911343,-0.368238,-0.324728,-0.777362,-0.274276,-0.284638,0.084677,0.352767,0.472696,0.12066199999999999,-0.597195,-0.21367600000000003 -486,1,0.039513,-0.03863,-0.037566,-0.087163,-0.796437,-0.300672,-0.136674,-0.553125,-0.7211350000000001,-0.996699,0.145616,-0.567723,0.092306,0.031601,-0.708116,-0.708046,-0.46301899999999996,-0.54191,-1.469938,-1.310945,-0.6668810000000001,-0.38256100000000004,-0.690597,-0.447891,-1.168707,-0.64542,-0.460674,-0.916831,-0.9689059999999999,-0.974339 -250,0,1.928106,0.215406,1.728734,1.985416,-0.49396899999999994,0.400354,2.048118,1.460136,0.364396,-0.30233899999999997,1.9349060000000002,0.9937389999999999,1.933096,2.016784,0.308838,1.066192,2.290035,2.1171919999999997,1.436213,-0.541186,2.161296,-0.718577,1.7355479999999999,2.1465509999999997,-0.586017,0.760202,2.098385,1.110141,0.419864,0.456586 -339,0,2.9821560000000003,0.8228129999999999,2.833789,3.5605059999999997,0.838642,0.086101,1.0060959999999999,1.435773,-0.497881,-0.36385,2.664822,1.1589610000000001,2.600377,3.106075,0.7500640000000001,0.454059,1.782814,2.375132,-0.053372,-1.0968870000000002,2.1793419999999997,-0.5304300000000001,1.780101,2.7229959999999997,-0.24966999999999998,-0.32237899999999997,0.130184,0.1969,-0.69648,-0.25604 -103,1,-1.140857,0.187723,-1.0437319999999999,-0.9130299999999999,1.03152,-0.153087,-0.037364999999999995,-0.26062199999999996,-0.450966,0.052877,-1.207427,0.02568,-1.15411,-1.014176,0.294605,-0.139691,-0.34224,-0.48052,0.486968,0.059878,-0.81161,0.009337999999999999,-0.66238,-0.628525,0.6723680000000001,-0.196627,-0.053174,-0.109676,-0.539078,-0.08507 -206,1,-1.211265,-0.400144,-1.196831,-0.9650639999999999,0.400283,-0.8242149999999999,-1.0077610000000001,-0.894201,0.14276,-0.562239,-1.207427,-0.46998599999999996,-1.196536,-1.022424,0.892395,-0.606845,-0.894408,-0.758321,0.44680699999999995,0.007427,-0.691063,0.227059,-0.667826,-0.616204,0.89271,-0.725901,-0.7930729999999999,-0.692495,0.188604,-0.50039 -345,1,-1.116007,-1.009179,-1.0833469999999998,-0.9202379999999999,0.159186,-0.576756,-0.962232,-1.124731,-0.755108,0.051769,-1.098366,-1.065715,-1.061432,-0.947908,0.175047,-0.24165,-0.665279,-0.736396,-0.652127,1.028102,-0.24243800000000001,2.344398,-0.233684,-0.462192,3.4398129999999996,1.186639,-0.191771,0.18879,1.962404,1.120795 -346,1,-0.544459,0.225177,-0.6171989999999999,-0.558987,-0.152049,-0.75742,-1.0896540000000001,-0.969572,-0.033578,-0.17266600000000001,-0.58714,-0.090674,-0.630582,-0.596093,-0.8895879999999999,-0.879367,-1.020585,-1.04288,-0.9369,-0.328545,-0.585313,-0.117666,-0.647034,-0.491014,0.046342,-0.811412,-0.864129,-1.0221049999999998,-0.08745499999999999,-0.393723 -317,0,1.1536170000000001,-0.11028199999999999,1.001959,1.0625120000000001,0.483572,0.140173,0.542655,0.959181,-0.143587,-0.10893699999999999,1.162387,-0.09765499999999999,1.101054,1.058322,0.078262,0.139464,0.30383899999999997,0.788802,-0.016862000000000002,-0.8729049999999999,-0.0038689999999999996,-1.209357,-0.157944,0.188399,-0.740023,-0.496754,-0.25908000000000003,-0.059391,-0.9434799999999999,-0.508333 -499,0,1.5719239999999999,0.827699,1.666184,1.5459379999999998,0.615079,0.670714,1.178807,1.472317,-0.68069,0.334943,1.8355009999999998,0.45386000000000004,1.8877869999999999,1.8916439999999999,0.863928,1.138207,1.632155,1.629686,0.132826,-0.08188200000000001,0.6685270000000001,-0.0015480000000000001,0.663312,0.764624,-0.124998,0.135916,0.289338,0.485634,-1.134784,-0.025685000000000003 -478,1,-0.801242,-0.615097,-0.751235,-0.725988,0.124117,-0.33884000000000003,-0.060394,-0.613574,0.065106,0.435246,-0.749028,-1.09364,-0.74056,-0.7109949999999999,0.586383,-0.418088,-0.448455,-0.7539359999999999,-0.11908900000000001,0.41711400000000004,-0.728238,-0.092266,-0.6430739999999999,-0.57198,-0.694688,-0.242457,0.32050500000000004,-0.6096060000000001,-0.255754,-0.068427 -47,0,-0.124082,0.370108,-0.132881,-0.213029,2.026595,1.032677,1.09581,1.434251,1.616558,1.8815990000000002,-0.271884,-0.14652300000000001,-0.246689,-0.342115,1.383436,0.355511,0.424367,0.631459,1.15509,0.704888,-0.426147,-0.5863109999999999,-0.47971400000000003,-0.353943,-0.169667,-0.11838199999999999,-0.094289,0.0574,-0.376832,-0.057458 -456,1,-0.652142,2.138591,-0.632092,-0.620162,0.360831,-0.325481,0.096485,-0.704325,-0.027107,-0.650904,-0.709266,2.3271490000000004,-0.704313,-0.6819850000000001,-0.198572,-0.352516,-0.215937,-0.741555,-0.04607,-0.161268,-0.330864,2.193808,-0.354471,-0.378585,0.940045,-0.07255199999999999,0.359962,-0.41868500000000003,0.509462,-0.630509 -211,1,-0.614867,-0.11191,-0.656516,-0.587641,-0.191501,-0.421538,-0.600115,-0.692448,-0.591712,-0.22254000000000002,-0.649623,-0.081366,-0.677951,-0.645296,-0.5444359999999999,-0.669764,-0.7797810000000001,-0.902509,-1.017221,-0.315787,-0.660385,-0.63802,-0.703963,-0.510815,-0.5080140000000001,-0.45874899999999996,-0.37944,-0.8190189999999999,-0.530602,-0.330177 -490,1,-0.43470600000000004,1.0279969999999998,-0.432526,-0.452984,-0.296707,-0.46988500000000005,-0.715735,-0.780458,0.322334,-0.10616700000000001,-0.5331779999999999,0.733108,-0.567973,-0.535798,-1.027649,-0.9919389999999999,-0.899681,-0.936557,-0.97706,-0.430613,-0.654249,-0.141253,-0.638124,-0.49057399999999995,-0.648353,-0.750492,-0.7454930000000001,-1.175231,-0.540288,-0.528003 -113,1,-1.058024,-0.47668,-1.0318180000000001,-0.889826,-0.10382899999999999,-0.314031,-0.684551,-0.810759,-0.837615,0.349906,-1.027362,0.20951799999999998,-0.9609280000000001,-0.9120729999999999,1.127241,0.491962,-0.301813,-0.47046000000000004,0.402996,2.12958,-0.258319,1.1668889999999998,-0.40843,-0.449431,1.6130719999999998,0.6702199999999999,0.464738,0.59107,0.281835,1.133656 -359,1,-0.879933,-0.107025,-0.9373959999999999,-0.7752100000000001,0.040829000000000004,-0.9501700000000001,-0.756994,-0.975815,-0.722753,-0.143295,-1.332393,-0.225644,-1.324225,-1.0702049999999999,0.323071,-0.848666,-0.774633,-0.899156,-1.115796,0.962892,0.370767,0.054696,0.198478,-0.216873,-0.068329,-0.9219620000000001,-0.278974,-0.848541,-0.13588599999999998,-0.40923200000000004 -344,1,-0.664567,-1.224133,-0.6880890000000001,-0.640202,0.597545,-0.908185,-0.7843399999999999,-0.547643,-0.219623,-0.326167,-0.686545,-0.8935120000000001,-0.6977220000000001,-0.667195,1.3265040000000001,-0.5975590000000001,-0.611921,-0.423515,0.7206279999999999,0.32071700000000003,-0.218617,-0.866083,-0.25299,-0.347783,0.831707,-0.8108529999999999,-0.5100779999999999,-0.095077,-0.12256800000000001,-0.262848 -532,1,-0.08680700000000001,-0.891932,-0.168624,-0.188419,-0.261639,-0.622558,-0.7272489999999999,-0.41973999999999995,-0.153294,-0.339467,-0.12703699999999998,-0.6887300000000001,-0.173371,-0.225791,-0.255504,-0.602486,-0.89491,-0.7768930000000001,-0.659428,-0.176862,-0.723546,-1.336724,-0.73911,-0.507955,-1.071036,-1.019992,-0.822815,-1.089909,-1.211063,-0.793535 -261,0,0.7415229999999999,0.943318,0.623679,0.5931489999999999,-0.366845,-0.672177,-0.724851,-0.49115299999999995,-0.725988,-1.0415860000000001,0.9152950000000001,0.8773860000000001,0.7838890000000001,0.7912600000000001,-0.693171,-0.7853680000000001,-0.751909,-0.530044,-0.904042,-1.37899,-0.016141,0.1817,-0.143093,0.089611,-0.43834399999999996,-0.805823,-0.644364,-0.669137,-0.778813,-0.8385469999999999 -4,0,1.298575,-1.4667700000000001,1.338539,1.220724,0.220556,-0.313395,0.613179,0.729259,-0.868353,-0.3971,1.7502970000000002,-1.151816,1.776573,1.826229,0.28037199999999995,0.5393399999999999,1.371011,1.428493,-0.009559999999999999,-0.56245,1.270543,-0.790244,1.273189,1.1903569999999999,1.483067,-0.04852,0.828471,1.144205,-0.361092,0.499328 -226,1,-0.983474,-0.9570690000000001,-1.0065,-0.8529100000000001,0.07589800000000001,-0.8840120000000001,-1.179224,-1.065348,-0.46229,-0.069592,-1.047243,-0.891185,-1.044132,-0.925156,0.636199,-0.513983,-1.03147,-0.9481649999999999,-0.08623,0.24133200000000002,-0.7719090000000001,-0.569982,-0.8207899999999999,-0.626544,-0.176,-0.9734360000000001,-0.964097,-1.106941,-0.668632,-0.36838000000000004 -464,1,-0.283535,-0.291038,-0.362232,-0.33924699999999997,-0.18273399999999998,-0.36746599999999996,-0.40581500000000004,-0.153883,-1.0770469999999999,-0.814381,-0.271884,-0.248914,-0.316713,-0.334435,-1.54431,-0.841464,-0.504827,-0.521532,-1.3056450000000002,-1.03593,-0.732208,-0.964965,-0.806929,-0.515876,-0.35734099999999996,-0.589531,-0.538925,-0.540667,-1.163843,-0.802991 -277,0,0.764302,-0.22427199999999997,0.647508,0.624792,-0.353694,-0.879559,-0.24557800000000002,0.22525900000000001,-0.539944,-1.472721,1.329956,0.16064900000000001,1.1916719999999998,1.2716290000000001,-0.50743,-0.862311,-0.107964,0.245323,-0.955155,-1.819865,-0.277447,-0.7055140000000001,-0.24903000000000003,-0.07914199999999999,0.176681,-0.8013520000000001,-0.18779200000000001,0.45968000000000003,-0.15041600000000002,-0.801478 -257,0,0.302508,-0.491336,0.37347800000000003,0.084761,1.9345400000000001,1.247056,0.8189930000000001,1.648946,0.577943,1.9480979999999999,0.338746,-0.46998599999999996,0.462606,0.166127,2.643065,2.351102,1.958584,1.941793,2.140845,1.865906,0.91684,-0.2864,0.591532,0.42073699999999997,1.036382,1.1402510000000001,0.5947140000000001,1.5367520000000001,-0.206112,1.8402290000000001 -260,0,1.6692529999999999,2.195586,1.639376,1.693603,0.8693270000000001,0.25595100000000004,0.511471,0.8388909999999999,0.40484000000000003,-0.21921500000000002,1.7559770000000001,1.808213,1.6859549999999999,1.800632,0.259022,0.084505,0.792227,1.145533,0.008694,-1.003325,-0.026968000000000002,-0.333573,-0.13814300000000002,0.264085,-0.66602,-0.542025,-0.355567,-0.585761,-0.9350040000000001,-0.719776 -301,1,-0.581734,-0.42457,-0.569839,-0.578851,-1.199727,-0.244691,-0.392382,-0.5840350000000001,-0.349045,-0.34944200000000003,-0.47353500000000004,0.139706,-0.47529499999999997,-0.522146,-0.8433299999999999,-0.055736,-0.257368,-0.46246400000000004,-0.11178699999999998,-0.043607,-0.147877,-0.320873,-0.142103,-0.264396,-0.17033299999999998,0.45895699999999995,0.504858,0.362354,0.8303200000000001,0.32382 -537,1,-0.681134,1.060565,-0.629709,-0.690654,1.9476900000000001,0.45060900000000004,-0.636576,0.24657600000000002,-0.158147,0.873031,-0.692225,1.198521,-0.6425270000000001,-0.706729,1.9385270000000001,0.9638540000000001,-0.548016,-0.09309400000000001,1.166043,1.595143,-0.395108,1.380982,-0.350511,-0.42654899999999996,1.946419,0.529378,-0.439122,0.951175,-0.598406,0.767508 -403,1,-0.49890100000000004,-0.432712,-0.523373,-0.526817,-0.664929,-0.371919,-0.437479,-0.46785600000000005,0.6410359999999999,-0.310651,-0.337208,-0.725963,-0.36202199999999995,-0.418905,0.17291199999999998,-0.302863,-0.7010609999999999,-0.645343,-0.27973000000000003,-0.113069,-0.9361280000000001,-0.565809,-0.9249940000000001,-0.637545,-1.384716,-0.705781,-0.522678,-0.728019,-0.223063,-0.689516 -233,0,1.698245,1.905725,1.651291,1.7428240000000002,-0.441366,0.138901,0.6832229999999999,0.634854,-0.750255,-0.036897000000000006,1.8127799999999998,1.982743,1.74774,1.8888,-0.339479,0.057973000000000004,0.8361700000000001,0.8893989999999999,-1.327551,-0.974973,0.428875,-0.050536000000000005,0.44599300000000003,0.652855,-0.673687,-0.271519,0.088738,-0.14374,-0.915632,-0.343416 -487,0,1.592632,0.767446,1.389175,1.51078,0.834259,0.7521399999999999,1.541979,1.391616,0.590885,0.340485,1.508885,-0.10929000000000001,1.488242,1.456496,0.892395,0.7667579999999999,1.717529,1.817982,0.041553,-0.233566,0.580101,0.346806,0.378669,0.602911,-0.58435,0.159389,0.353662,-0.005862,-0.40831300000000004,-0.220862 -141,0,0.756019,-0.066314,0.647508,0.619518,-0.042459,-0.195073,0.038435000000000004,0.106491,-0.17594300000000002,-0.131104,0.563117,-0.288475,0.540867,0.44968400000000003,0.060471000000000004,0.177367,0.07119500000000001,0.271117,0.180289,-0.045025,1.0817809999999999,0.20891500000000002,0.8251870000000001,0.742402,-0.09033,-0.34082199999999996,-0.040575,-0.020461,-0.217009,-0.136512 -68,1,-1.2340440000000001,-0.492965,-1.2438930000000001,-0.977194,0.693984,1.159269,4.700669,0.919592,2.14719,1.859432,-1.4479870000000001,-0.45602299999999996,-1.366651,-1.150124,0.7287140000000001,0.7004279999999999,2.814833,-0.13333299999999998,1.093024,2.503828,-0.280696,-0.041464,-0.485654,-0.49871400000000005,0.836041,3.3858919999999997,9.015603,3.475158,2.594434,2.1802770000000002 -363,1,0.385341,-0.037002,0.296035,0.225746,0.062747,-0.5494020000000001,-0.5080020000000001,-0.35594000000000003,-0.8198200000000001,-1.067077,0.673883,-0.232625,0.6026520000000001,0.521071,0.035563,-0.372605,-0.3789,-0.014681,-1.155957,-0.973555,-0.23919,0.40305,-0.25843499999999997,-0.148667,0.07201,-0.415155,-0.450064,-0.23782199999999998,-0.43616099999999997,-0.678547 -379,0,-0.627292,1.163157,-0.461717,-0.6547930000000001,3.771263,4.348873,2.7250490000000003,2.09813,2.027474,3.122913,-0.865474,-0.10696300000000002,-0.768981,-0.834144,1.796196,2.104733,1.005662,0.380483,1.403355,2.367738,-0.699364,-0.344459,-0.567829,-0.579681,0.12134500000000001,1.118454,0.46374399999999993,0.25854,-0.382886,0.242117 -51,1,-0.331164,-0.405029,-0.333042,-0.393567,-1.028767,-0.611108,-0.802091,-0.43770699999999996,-0.897473,-0.20480700000000002,-0.138398,-0.686403,-0.196026,-0.236314,-1.388458,-0.8291459999999999,-0.881727,-0.817389,-1.674391,-0.463218,-0.7867069999999999,-0.532425,-0.701488,-0.5673600000000001,-0.8546940000000001,-0.766142,-0.6997359999999999,-0.622907,-0.882941,-0.470508 -195,1,-0.494759,-0.598813,-0.49001300000000003,-0.49218599999999996,-0.993698,-0.659455,-0.45954799999999996,-0.49115299999999995,0.199382,-0.801081,-0.34572800000000004,-0.6887300000000001,-0.38879600000000003,-0.393877,-1.206274,-0.9604799999999999,-0.6286189999999999,-0.6486970000000001,0.063458,-0.8686520000000001,-0.761442,-0.5592779999999999,-0.679706,-0.5409579999999999,-0.581017,-0.536994,-0.287926,-0.547642,-0.243646,-0.6214310000000001 -239,0,1.292363,3.125425,1.010895,0.9271530000000001,0.181104,0.758501,0.249047,1.400752,-0.07725800000000001,0.056201999999999995,0.946536,4.651889,0.8827459999999999,0.755709,0.12523099999999998,0.48248599999999997,0.664167,1.010888,-0.009559999999999999,-0.44478900000000005,0.474351,-0.6545310000000001,0.06729500000000001,0.190599,-0.7270220000000001,0.13256199999999999,-0.19475499999999998,0.315313,-1.002808,-0.391832 -161,0,1.192963,-1.281128,1.1717389999999999,1.080091,-0.875341,-0.335023,-0.219671,0.9607040000000001,-0.740548,-1.187883,1.437881,-0.779486,1.414099,1.4280549999999999,-0.6703979999999999,0.268334,0.38293499999999997,1.2316850000000001,-0.257825,-1.5646959999999999,2.146859,-1.058222,2.0320709999999997,1.7373189999999998,0.7883720000000001,0.283464,0.38284,2.618691,1.334007,-0.163746 -169,1,-0.032964999999999994,-0.43596899999999994,-0.079266,-0.15273299999999998,-0.472051,-0.57612,-0.986891,-0.453086,-0.803642,-1.089797,0.239341,-0.544452,0.175098,0.088198,0.155832,-0.483092,-0.788192,-0.286549,-0.11543800000000001,-0.892752,-0.48317200000000005,0.000266,-0.48169399999999996,-0.353283,-0.653686,-0.658275,-0.81625,-0.441719,-0.852671,-0.790131 -212,0,2.452025,-1.1736520000000001,2.419765,2.845036,-0.796437,-0.653093,0.229857,0.683579,-2.026684,-1.5902020000000001,3.971288,-0.19073800000000002,3.9761300000000004,5.244841,1.269571,0.8956280000000001,2.903973,2.852321,-0.597362,-1.069952,8.906908999999999,0.470181,9.461986,10.676614,2.136427,0.125297,1.060902,0.368843,3.303954,0.257626 -120,1,-0.714267,-1.580761,-0.700599,-0.650574,0.983301,-0.097107,-0.29739099999999996,-0.381064,0.18644000000000002,0.071164,-0.771749,-1.9709439999999998,-0.767334,-0.715545,-0.187185,-0.7105100000000001,-0.673942,-0.5852430000000001,-0.527994,-0.23640100000000003,-0.954174,-1.3719219999999999,-0.872768,-0.656467,-0.333673,-0.5694100000000001,-0.555503,-0.8655729999999999,-0.859936,-0.601005 -367,1,-0.40985600000000005,-0.266612,-0.399464,-0.449996,0.19425499999999998,-0.237058,-0.14818800000000001,-0.40070700000000004,0.513232,-0.512365,-0.544538,-0.295456,-0.5626180000000001,-0.5588350000000001,-0.28824,-0.6176470000000001,-0.563459,-0.7389760000000001,-0.42576800000000004,-0.5156689999999999,-0.550304,-0.795142,-0.4911,-0.47891300000000003,-0.40267600000000003,-0.648215,-0.39436,-0.763868,-0.140729,-0.694055 -171,0,0.354279,0.682768,0.27816399999999997,0.198674,0.338913,-0.634009,-0.037364999999999995,0.021222,-0.027107,-0.567226,-0.198041,0.079202,-0.252456,-0.25451599999999996,-0.418473,-0.785747,-0.37940300000000005,-0.37502199999999997,-0.77991,-0.8629819999999999,0.232895,-0.12673800000000002,0.136599,0.067389,-0.346007,-0.8298559999999999,-0.34429299999999996,-0.381702,-0.646838,-0.728854 -348,1,-0.778463,-0.795854,-0.8218270000000001,-0.711573,0.9087799999999999,-0.9050040000000001,-0.83457,-0.748025,-0.08049400000000001,0.20416199999999998,-0.754709,-0.758542,-0.780514,-0.717252,-0.398546,-0.8619319999999999,-0.790076,-0.662883,-0.648476,0.13075799999999999,-0.846259,-0.826168,-0.8792040000000001,-0.6179640000000001,0.716703,-0.946218,-0.745825,-0.889093,-0.36230300000000004,-0.297269 -409,1,-0.449201,0.521552,-0.543926,-0.47531,-0.366845,-0.47561000000000003,-0.645212,-0.293664,0.8966459999999999,-0.830451,-0.527497,-0.318726,-0.558499,-0.536936,-0.6768029999999999,-0.740642,-0.711733,-0.577505,0.563638,-0.967885,-0.256153,1.023556,-0.38961799999999996,-0.32006100000000004,-0.384342,-0.132913,-0.374798,-0.169694,0.634173,-0.577932 -112,1,-0.200702,-0.31709299999999996,-0.00778,-0.301628,-1.879621,1.0498530000000001,1.94833,0.5465399999999999,-0.813348,1.344065,0.037691,0.083856,0.241414,-0.071072,-1.280286,2.254449,2.655385,0.749595,-0.39291,2.111151,-0.15292999999999998,0.49558199999999997,0.263822,-0.24393499999999999,-0.581017,2.737574,3.700532,1.804398,0.619644,3.474275 -524,1,-1.041457,-0.43759799999999993,-0.981182,-0.886134,0.417818,-0.19062,-0.901208,-0.751071,-0.645099,0.451317,-1.2156639999999999,-0.839989,-1.1932399999999999,-1.028681,-0.102498,-0.381891,-0.822342,-0.638637,-1.550259,0.8664950000000001,-0.56077,-0.0015480000000000001,-0.44060699999999997,-0.552179,0.563697,-0.282697,-0.7053729999999999,-0.8102600000000001,-0.27875900000000003,-0.113817 -62,0,0.290083,0.6241439999999999,0.352628,0.138729,1.386591,2.356484,2.015015,0.972885,-0.091818,1.6211440000000001,0.034851,0.665623,0.183336,-0.026135000000000002,0.607733,1.828041,1.565613,0.970134,0.501572,1.434954,1.077089,0.092797,1.241012,0.449779,0.788706,2.519604,1.230002,1.084187,-0.428896,0.8768229999999999 -393,0,2.06271,0.498754,1.9282990000000002,2.1102279999999998,0.781656,2.01933,2.0778630000000002,1.9428189999999999,1.500078,0.9284469999999999,2.125195,0.695875,2.159642,2.139081,1.4474850000000001,1.9777580000000001,2.413074,2.7672,1.279223,0.462478,0.790518,-0.536961,0.63955,0.8944340000000001,-0.608684,0.658483,0.506848,0.175813,-0.011175,0.08816900000000001 -75,0,0.724957,-0.18193299999999998,0.641551,0.601939,0.772888,-0.316575,0.051389,0.56938,-0.405668,-1.1125180000000001,0.5517569999999999,0.083856,0.49967700000000004,0.46305100000000005,-0.333074,-0.380943,0.111622,0.45038500000000004,-0.049721,-1.259911,1.2351729999999999,-0.364417,1.070722,0.856151,1.259725,-0.192715,0.102995,1.020925,-0.610514,-0.6985939999999999 -124,1,-0.41606800000000005,-0.47668,-0.454866,-0.436812,-1.309316,-0.007411,0.28119,-0.378019,-1.379572,-0.424808,-0.21508200000000002,-0.674768,-0.241747,-0.28836100000000003,-1.794101,-0.58922,-0.098925,-0.539588,-1.422476,-0.647506,-0.8708020000000001,-0.139439,-0.813365,-0.56494,-0.374008,0.40362600000000004,0.586093,-0.22971100000000003,-1.024602,0.106325 -231,1,-0.867508,1.314602,-0.8173600000000001,-0.752884,-1.7682779999999998,-0.706529,-0.78338,-1.427893,-0.083729,-0.724607,-0.797311,1.8128669999999998,-0.832414,-0.73716,-1.9592049999999999,-1.254796,-0.90985,-1.1812129999999998,0.20949600000000002,-0.923939,-1.0256370000000001,-0.588126,-0.89455,-0.69816,-1.129372,-0.5034609999999999,-0.516046,-1.406542,-0.626254,-0.6592560000000001 -285,1,-0.573451,-0.422942,-0.646389,-0.55635,-1.25233,-1.196102,-1.279065,-1.6114950000000001,-0.640246,-1.088135,-0.439453,-0.207027,-0.500009,-0.47180600000000006,-0.884606,-1.1784219999999999,-1.091521,-1.186398,-0.418466,-0.602143,-0.48100699999999996,0.24157399999999998,-0.566839,-0.393546,-0.219335,-0.976398,-0.9958290000000001,-1.4391459999999998,0.625698,-0.673251 -448,1,0.006379999999999999,0.441759,0.024984,-0.088042,-1.028767,0.06765299999999999,0.507153,-0.284223,-0.69525,-0.5162439999999999,0.114375,0.011717,0.093953,0.013683,-0.888165,-0.499201,-0.007901,-0.507346,-1.236277,-0.756662,-0.547416,0.248831,-0.431696,-0.380565,-0.964699,-0.279344,0.062875,-0.478216,-0.799396,-0.555237 -460,0,1.38555,1.435106,1.335561,1.3490520000000001,1.211247,-0.06275499999999999,-0.039764,0.6226729999999999,0.176733,0.368747,0.8386110000000001,1.829157,0.792127,0.785003,0.186433,0.126198,0.14941300000000002,0.396991,-0.067976,0.001756,1.890966,-0.117666,1.576643,1.647112,0.5663630000000001,-0.183773,-0.155298,0.45156899999999994,-0.011175,0.23530900000000002 -218,0,1.959169,0.48246999999999995,1.877663,1.9836580000000001,0.128501,0.440431,0.42943400000000004,1.026178,0.2738,-0.07735,1.61113,0.5283260000000001,1.5541459999999998,1.635674,-0.180069,0.497647,0.48211899999999996,0.979935,1.030958,-0.990566,1.985527,-0.055978999999999994,1.792477,1.8495279999999998,-0.078996,0.34717800000000004,0.08509,0.863582,-0.190372,0.094599 -467,1,-1.060095,-0.17216199999999998,-1.076794,-0.8796299999999999,0.281926,-0.819126,-0.9983569999999999,-1.364398,0.252769,-0.28793,-1.266502,-0.276839,-1.273149,-1.0483049999999998,-0.9429620000000001,-0.94873,-0.929185,-1.113014,-0.48053199999999996,0.18746300000000002,-0.22944499999999998,0.172629,-0.292592,-0.425889,1.3130600000000001,-0.72143,-0.7156520000000001,-1.275315,0.773413,-0.30861700000000003 -525,1,-1.4073719999999998,-1.176908,-1.3094219999999999,-1.063508,1.390974,-0.195709,-0.464345,-0.448975,0.133053,1.161193,-1.578066,-1.440373,-1.5421209999999999,-1.233172,0.5152180000000001,-0.531039,-0.792838,-0.8723299999999999,-0.487834,1.199632,-1.005064,-0.9753069999999999,-0.889599,-0.727884,0.285352,-0.439188,-0.460343,-0.726073,-0.559661,0.049208999999999996 -80,1,-0.654213,1.05568,-0.677068,-0.624908,1.022753,-0.551311,-0.46386499999999997,-0.812129,-0.224476,0.25292800000000004,-0.760389,0.391029,-0.747974,-0.720665,0.9849100000000001,-0.203179,-0.538474,-0.68584,0.110921,1.028102,-0.288997,1.736593,-0.390608,-0.345803,1.1097190000000001,-0.47048599999999996,-0.20006,-0.695902,-0.288445,0.0685 -472,1,0.188613,-1.2143620000000002,0.141149,0.045735000000000005,-1.133973,0.157985,0.205869,0.001428,-0.344192,-0.067376,0.22514,-1.014519,0.184572,0.091042,-1.094545,-0.357254,-0.41945299999999996,-0.43099499999999996,-0.45497600000000005,-0.8658170000000001,-0.579538,-1.421454,-0.514861,-0.374625,-1.2567110000000001,-0.434717,-0.291573,-0.551048,-1.09725,-0.535568 -549,1,-0.67078,0.940061,-0.695833,-0.659188,-0.524654,-0.578665,-1.008672,-1.248067,0.25600500000000004,-0.42591599999999996,-0.939318,1.1449989999999999,-0.95063,-0.834144,-1.027649,-0.7262390000000001,-0.920522,-1.051341,0.600147,0.068384,0.41299399999999997,1.2721209999999998,0.345502,-0.161428,0.407357,-0.378827,-0.6340859999999999,-0.9536530000000001,0.498565,-0.309373 -527,1,-0.550672,-1.043377,-0.5969439999999999,-0.5549430000000001,-0.138898,-0.298127,-0.44659399999999994,-0.115817,0.338511,-0.44475699999999996,-0.507616,-1.633519,-0.536668,-0.5301100000000001,-0.45049700000000004,-0.782146,-0.7434970000000001,-0.5790529999999999,-0.44767399999999996,-0.66877,-1.041517,-1.3084200000000001,-1.03692,-0.690459,-1.1200379999999999,-0.91151,-0.7679739999999999,-0.9823639999999999,-1.120254,-0.9187360000000001 -168,0,1.422825,1.0833629999999999,1.430876,1.370147,0.229323,0.818934,1.040158,0.8754350000000001,-1.19838,0.501745,0.949377,1.254371,0.993959,0.937732,0.607733,1.060506,1.595745,1.428493,-0.998966,0.12083499999999998,2.464469,0.350434,2.213253,1.8033240000000001,-0.289005,0.607064,0.482311,0.631622,-1.100882,0.501219 -351,0,0.225888,-0.245442,0.361564,0.06102899999999999,0.9920680000000001,1.59248,1.9910279999999998,1.505816,2.174692,1.166735,0.460872,-0.016208,0.6232479999999999,0.294964,1.9883419999999998,2.502714,2.543646,1.941793,2.056873,1.8758290000000002,0.415882,0.194401,0.30243400000000004,0.239443,0.762705,2.241833,2.2426180000000002,1.7897990000000001,4.224151,1.3371549999999999 -57,0,0.3315,0.817928,0.25135599999999997,0.184435,0.19425499999999998,1.111558,0.415041,1.047495,1.289766,1.410564,0.165497,0.535307,0.147501,0.005719,1.233989,0.6094609999999999,0.5084850000000001,0.833426,0.786345,0.677954,0.062901,-0.121295,-0.06487799999999999,-0.005436,-1.127372,0.171685,-0.204702,0.149859,-0.287234,0.11842899999999999 -560,1,-0.200702,1.2201520000000001,-0.21032399999999998,-0.305671,-0.36246100000000003,-0.177261,-0.669679,-0.149315,-1.0527799999999998,-0.040776,-0.021952,1.829157,-0.024262,-0.154973,0.20849499999999999,0.156521,-0.55467,-0.151647,-1.002617,-0.15417999999999998,-0.146794,0.49921099999999996,0.010860999999999999,-0.230954,0.071677,0.07276,-0.370819,0.724082,0.031201999999999997,0.570818 -424,1,-1.04767,-0.40828600000000004,-1.05654,-0.8783989999999999,0.325762,-0.75742,-1.205802,-1.046314,0.47763999999999995,-0.213673,-1.245485,-0.039478,-1.237314,-1.038635,0.792763,-0.39818899999999996,-1.002707,-0.754452,2.651977,1.062124,1.051464,0.9618680000000001,0.861819,0.07003,2.0097549999999997,-0.370444,-0.8578620000000001,-0.207002,1.266203,0.162684 -355,1,-0.600372,-0.52879,-0.54333,-0.585707,-0.998082,-0.343929,-0.160182,-0.33431900000000003,-1.261474,-0.668637,-0.445133,-0.051114,-0.413922,-0.480907,-0.623429,-0.010251999999999999,0.178289,-0.129206,-1.017221,-0.135751,-0.162314,0.47381,0.171251,-0.282658,0.937378,0.942961,1.021776,1.275595,0.7080310000000001,0.155119 -42,0,1.619553,1.2201520000000001,2.089143,1.354326,-0.33616,3.117943,2.168536,2.050928,2.8622490000000003,1.100236,1.403799,1.2846229999999998,1.49648,1.277317,-0.394988,2.1729580000000004,1.53046,1.307519,1.819562,0.089648,2.078645,0.814906,2.9523330000000003,1.420495,-0.16433299999999998,4.198526,2.166357,2.365644,3.9698860000000002,1.456682 -217,1,-0.991758,-0.196589,-0.9490129999999999,-0.838319,-1.62625,-0.7287939999999999,-0.382307,-1.2013209999999999,-0.052992,-0.32450500000000004,-1.115407,-0.421117,-1.1088,-0.949046,-1.1258569999999999,-0.857952,-0.389949,-0.9855659999999999,0.5563359999999999,0.049955,-0.174946,1.2793780000000001,-0.058938,-0.38606599999999996,-0.787025,0.319792,0.856986,-0.176182,0.612379,0.309446 -286,1,-0.627292,0.262631,-0.448611,-0.587992,-0.91041,0.17198,-0.171216,0.013609,-0.704957,0.879127,-0.621222,0.342161,-0.580742,-0.608323,-0.7337359999999999,-0.061421,-0.289509,-0.28397,-0.816419,0.689294,-0.472706,0.31414699999999995,0.164321,-0.405427,-0.107331,1.457703,0.397098,0.735436,-0.22064099999999998,1.596257 -135,0,-0.36843899999999996,1.252721,-0.45337700000000003,-0.399017,0.417818,-0.64864,-0.26141,-0.32426900000000003,-0.116085,-0.181532,-0.38549,0.740089,-0.42216000000000004,-0.42260299999999995,-0.413491,-0.8856209999999999,-0.523408,-0.56435,-0.8273719999999999,-0.304446,-0.608051,0.296004,-0.697527,-0.45031099999999996,0.15267999999999998,-0.752169,-0.284279,-0.466537,-0.493068,-0.44100500000000004 -521,0,2.826844,0.204007,2.932082,3.0964169999999998,0.08028099999999999,1.0466719999999998,0.9288559999999999,2.02352,0.414547,0.707338,2.982918,0.5376340000000001,3.028755,3.373421,0.47251899999999997,2.013766,1.785325,2.532475,0.654911,0.651019,2.116181,-0.574155,2.071179,2.190554,-0.684021,0.371211,0.12653699999999998,0.677041,-0.212166,0.36504899999999996 -278,1,-0.15928599999999998,0.068847,-0.248748,-0.24871500000000002,-1.199727,-1.132615,-0.797293,-0.955564,-0.915269,-1.181233,-0.152598,-0.337343,-0.23598000000000002,-0.23489200000000002,-1.201293,-1.209502,-0.86415,-0.9424899999999999,-0.871183,-1.07704,-0.531175,-0.092266,-0.585651,-0.398607,-1.1000379999999998,-1.1292,-0.704378,-1.0952620000000002,-0.859936,-1.009516 -504,1,-1.240257,-1.513995,-1.138153,-1.020263,2.535091,0.571476,-0.29883000000000004,-0.184337,0.222031,2.286134,-1.380108,-1.493895,-1.2554370000000001,-1.155243,4.770911,2.26582,0.106726,0.09287999999999999,2.0678259999999997,4.5678589999999994,0.008763,-0.22471300000000002,0.07323500000000001,-0.44656999999999997,0.9140440000000001,1.114541,0.097358,1.6357,0.590585,2.517299 -97,1,-1.107724,0.099787,-1.145302,-0.904416,-0.033692,-1.014866,-1.207529,-1.3825180000000001,-1.563999,0.32884800000000003,-1.232704,0.151341,-1.2299,-1.0249840000000001,0.42981899999999995,-0.9727979999999999,-1.0291350000000001,-1.057093,-1.685343,0.8650770000000001,-0.253266,1.4989139999999999,-0.36338200000000004,-0.44635,1.363062,-0.606297,-0.881503,-1.061847,-0.306607,0.708123 -84,1,-0.538247,-0.126566,-0.580264,-0.54967,0.23370700000000003,-0.34329299999999996,-0.218232,-0.582969,0.773694,-0.260777,-0.6041810000000001,-0.8469709999999999,-0.618637,-0.601781,0.061894000000000005,-0.6195430000000001,-0.593716,-0.781278,0.976194,-0.441954,-0.642699,0.069211,-0.705448,-0.531937,-0.35734099999999996,-0.411243,-0.39203899999999997,-0.7736,-0.099563,-0.449326 -553,1,-1.330337,-0.102139,-1.322527,-1.027998,-0.9673959999999999,-1.089612,-0.922365,-1.354653,-0.753491,-0.5550350000000001,-1.3616469999999998,0.616755,-1.357589,-1.1117290000000002,-0.281835,-0.915186,-0.613176,-0.931141,-0.436721,0.41995,-0.374896,1.201362,-0.368827,-0.49453400000000003,1.299726,-0.398947,0.26745399999999997,0.16608,2.06411,0.313228 -469,1,-0.602442,-0.045144,-0.569541,-0.619635,2.000293,0.213329,0.22266,0.41102299999999997,-0.38949,0.48512,-0.712106,-0.258223,-0.642115,-0.6999029999999999,1.5044170000000001,0.8330879999999999,0.165734,0.17335799999999998,0.530779,1.382503,0.017786,0.949168,0.079671,-0.274737,2.5164419999999996,0.367857,0.5874189999999999,1.072832,-0.299342,0.53791 -87,0,1.716882,0.770703,1.3593899999999999,1.305104,-0.327392,0.421983,1.455144,1.23326,1.707154,0.495095,1.389599,1.233427,1.236982,1.197683,-0.43199399999999993,0.308132,0.728197,0.8716010000000001,0.516175,-0.9225209999999999,0.52091,-1.003792,0.09353099999999999,0.380914,-1.0563690000000001,-0.39447600000000005,0.172625,0.033069,-0.109249,-0.173203 -22,0,0.37291599999999997,-1.074317,0.531343,0.176348,0.290694,2.170095,1.719008,1.8986619999999998,2.857396,0.8597309999999999,0.344426,-1.170433,0.43377299999999996,0.14081400000000002,0.77853,2.068725,1.492795,1.254641,2.589911,1.0663770000000001,0.12137,-0.9203319999999999,0.256397,0.10061200000000001,-0.083996,1.553833,1.079801,1.739514,1.9587720000000002,0.226609 -19,1,-0.240048,-1.045005,-0.225217,-0.297761,0.5098729999999999,-0.489605,-0.159223,0.216123,0.123347,-0.629292,-0.166799,-1.147162,-0.185728,-0.25195700000000004,0.101747,-0.43685,-0.27821,-0.028609,0.267911,-0.72831,-0.488225,-0.776999,-0.400014,-0.36912399999999995,0.47369300000000003,-0.607974,-0.266043,0.21961,-0.089876,-0.565449 -265,0,3.3590459999999998,3.4983370000000003,3.179304,4.485168,0.338913,0.064472,0.345477,0.78103,-0.052992,-0.097854,1.8752630000000001,2.753002,1.8012869999999999,2.1732099999999996,-0.11886600000000001,0.18873800000000002,0.601392,0.9683280000000001,-0.15559800000000001,-0.8587290000000001,2.767643,0.726003,2.417205,3.506257,-0.8300270000000001,-0.5979140000000001,-0.346946,-0.408142,-0.8320879999999999,-0.565828 -38,0,-0.264898,-0.077713,-0.34912600000000005,-0.31955900000000004,-1.6876200000000001,-1.2910780000000002,-1.1907860000000001,-1.303644,-2.1609599999999998,-1.601839,0.24502100000000002,1.375378,0.147089,0.12488699999999998,-0.177222,-1.005016,-0.813805,-0.5140520000000001,-0.9003909999999999,-1.0997219999999999,2.9192299999999998,1.7619939999999998,2.579575,1.444697,-0.052662,-0.81253,-0.45470600000000005,1.196112,-1.5328899999999999,-0.771975 -479,0,0.2321,-0.42782700000000007,0.441986,0.103922,0.23370700000000003,1.220974,1.523269,0.957658,0.67501,0.41086300000000003,0.6028789999999999,0.051276999999999996,0.7344609999999999,0.457647,0.44405200000000006,1.610099,1.692419,1.109679,1.239062,0.422785,-0.326533,-0.419392,0.100957,-0.158788,0.718703,1.6331959999999999,1.621255,1.679497,0.870276,0.96836 -17,0,0.971385,0.944946,0.8798379999999999,0.763667,2.039746,1.0752979999999999,0.9893049999999999,1.411411,1.3027090000000001,1.6765599999999998,0.5687979999999999,0.323544,0.664438,0.40929699999999997,1.4688350000000001,1.854573,1.047093,1.389802,1.286524,1.5256809999999998,0.592011,-0.261,0.48906099999999997,0.304568,-0.004993,-0.026164,-0.000455,0.190412,-0.44221499999999997,0.13129000000000002 -189,1,-0.604513,-0.991267,-0.6139220000000001,-0.586937,-0.998082,-0.56785,-0.623143,-1.011902,-0.677455,-1.157405,-0.518977,-0.788794,-0.541199,-0.5437609999999999,-1.1073540000000002,-0.602865,-0.63226,-0.8351870000000001,-0.527994,-1.14225,-0.602637,-0.691906,-0.58367,-0.48441300000000004,-0.34834,-0.187685,-0.355567,-0.8699530000000001,-0.035391000000000006,-0.782944 -178,1,-0.46991000000000005,0.54435,-0.56835,-0.477771,-2.240829,-1.399158,-1.2675610000000002,-1.60408,-0.97998,-1.41398,-0.317327,0.681912,-0.41021499999999994,-0.36543600000000004,-2.40897,-1.610136,-1.094848,-1.214049,-1.521051,-1.482475,-0.8375969999999999,-0.13581,-0.873758,-0.57198,-1.207709,-1.298098,-1.004615,-1.613035,-0.534235,-1.069167 -164,0,2.431317,0.414075,2.291686,2.6762759999999997,-0.41944799999999993,0.661808,0.588232,1.8270970000000002,1.113428,0.439125,2.596659,0.640025,2.476807,2.932585,-0.8518700000000001,0.192528,0.547405,1.240713,-0.038768000000000004,-1.030259,0.9348860000000001,-0.6545310000000001,0.8598389999999999,1.265383,-0.7103550000000001,-0.002132,-0.12180899999999999,0.7192149999999999,-0.11893499999999999,-0.020766999999999997 -451,0,1.070784,0.860267,0.969195,0.9500059999999999,0.8956290000000001,-0.44380299999999995,0.602144,0.48715600000000003,-0.983215,-1.276549,1.551487,1.328837,1.471766,1.5247540000000002,0.486752,-0.106715,0.962975,1.0758889999999999,-0.5425979999999999,-1.259911,0.22459400000000002,0.286932,0.024722,0.348571,1.619739,-0.345852,0.5695140000000001,0.518075,-0.500333,-0.751927 -290,1,-0.103373,-0.577643,-0.16564500000000001,-0.199142,-1.426358,-0.044944,-0.24078000000000002,-0.19042699999999999,-1.017189,0.22411199999999998,0.08029299999999999,0.10247300000000001,0.167272,-0.011061,-0.625564,1.198852,0.595114,0.44109899999999996,-0.3564,1.293194,1.7177240000000003,1.003598,0.739546,0.809067,0.24035,4.522686,2.244939,2.581382,2.4491400000000003,7.211398 -67,1,-0.8157369999999999,-0.29918,-0.8715700000000001,-0.7275699999999999,-0.14766500000000002,-1.03554,-0.6130680000000001,-0.685139,-0.810113,-0.9717620000000001,-0.8001510000000001,-0.05809500000000001,-0.830767,-0.741711,-1.065367,-1.086507,-0.649209,-0.6866140000000001,-1.079287,-0.8686520000000001,-0.47811899999999996,-0.497046,-0.512386,-0.488154,0.747037,-0.908883,-0.373471,-0.45923800000000004,0.15591300000000002,-0.6237 -110,1,-1.080803,-0.68512,-1.059816,-0.902834,0.6282300000000001,-0.49469399999999997,-0.682153,-0.932876,-0.5949479999999999,0.040685,-1.2355450000000001,-0.535144,-1.213835,-1.037213,0.522334,-0.384734,-0.5707399999999999,-0.803203,-0.831023,1.113158,-0.007839,0.375835,-0.058938,-0.384306,2.269766,0.21472,-0.154967,-0.25404299999999996,1.485355,0.318902 -556,1,-1.1636360000000001,-0.45551,-1.173002,-0.937465,-0.257255,-0.8541129999999999,-1.257616,-1.405205,-1.033367,-0.915792,-1.126767,0.06989400000000001,-1.121981,-0.976065,0.28037199999999995,-0.555297,-1.051784,-0.973959,-0.075277,0.07263700000000001,-0.581343,1.584188,-0.6029770000000001,-0.5178560000000001,1.95642,-0.182096,-0.9191030000000001,-0.764679,0.626908,-0.573771 -150,1,-0.436776,-0.255213,-0.48971499999999996,-0.463884,-0.11698,-0.9145469999999999,-0.9166559999999999,-0.786396,0.47763999999999995,-1.085918,-0.32016700000000003,0.346815,-0.348429,-0.385345,1.2197559999999998,-0.539188,-0.7211489999999999,-0.579569,2.659279,-0.27325900000000003,0.054238999999999996,0.190772,0.003436,-0.12226500000000001,-0.007993,-0.7857029999999999,-0.41126999999999997,-0.04317,1.085796,-0.855568 -321,0,1.406258,-0.431084,1.278968,1.364873,-1.182192,-0.639734,-0.055596000000000007,0.424727,0.249533,-1.364106,1.713375,0.08618300000000001,1.611812,1.7608150000000002,-1.1500540000000001,-0.35441100000000003,0.335226,0.731023,0.42490200000000006,-1.678104,0.676106,-0.962606,0.49599099999999996,0.759343,-0.8350270000000001,-0.654921,-0.180497,0.10930699999999999,0.168021,-0.8343860000000001 -526,1,-0.190348,-0.084227,-0.15968800000000002,-0.282643,1.316453,0.36982,-0.032568,0.42777299999999996,0.998566,0.149854,-0.18952,-0.12558,-0.186552,-0.295187,0.792763,0.17926199999999998,-0.587439,-0.44879300000000005,-0.323542,0.05279,-0.7412310000000001,-1.106847,-0.704458,-0.533917,-0.876028,-0.617475,-0.557161,-0.718124,-0.847828,-0.7594920000000001 -18,0,2.28843,0.84724,2.369129,2.6674860000000002,0.825491,0.386359,1.271399,1.8910490000000002,-0.21477,-0.432012,1.6139700000000001,0.665623,1.566503,1.720997,0.138753,-0.031099,0.742007,1.188093,-0.838325,-1.254241,1.274152,-0.362603,1.484567,1.585507,-0.182334,-0.36597199999999996,0.066854,0.553762,-0.845406,-0.68006 -446,0,1.089422,2.094623,1.135996,0.978132,0.338913,0.7756770000000001,1.764105,1.254577,0.115258,0.37705900000000003,1.028901,2.033938,1.043388,0.929199,0.256887,0.512808,1.016961,0.877275,-0.360051,-0.5156689999999999,-0.055841999999999996,-0.253742,0.003436,0.07948999999999999,-0.775691,-0.29778699999999997,0.16831400000000002,-0.11292,-0.98949,-0.468617 -432,0,1.192963,-0.098883,1.153867,1.051965,1.496181,0.254042,1.240695,1.563677,0.21232399999999998,-0.177099,1.719055,0.058258000000000004,1.723026,1.6925560000000002,1.205523,0.844459,1.563102,1.985643,-0.319891,-0.321457,0.100798,-0.39163200000000004,0.070265,0.267385,0.6820350000000001,0.09344,0.781388,1.184757,0.48040299999999997,0.07946900000000001 -523,1,-0.240048,-0.007690000000000001,-0.23325900000000002,-0.314109,0.444119,0.014854,-0.37751,0.21003200000000002,-0.083729,0.35267600000000005,-0.11851700000000001,-0.141869,-0.133416,-0.238589,0.199243,0.050392,-0.438788,-0.286033,-0.3564,0.79845,-0.310652,0.058325,-0.288137,-0.305539,-0.100664,-0.165888,-0.365845,0.279627,-0.223063,-0.018119999999999997 -418,1,-0.542388,-1.426059,-0.570137,-0.551428,-0.042459,-0.595204,-0.855583,-0.49282700000000007,-0.20344500000000001,0.592627,-0.405371,-1.6567900000000002,-0.45676000000000005,-0.454741,-0.605638,-0.879367,-0.818576,-0.6422479999999999,-0.834674,-0.006749,-0.649196,-1.036269,-0.662875,-0.505315,-0.30333899999999997,-0.7180770000000001,-0.756601,-0.5783,-0.413156,0.23417399999999997 -272,0,2.4685919999999997,0.407562,2.640181,2.642876,-0.22657,1.388279,1.480091,1.058154,-0.109614,0.256807,2.164957,0.395683,2.2790939999999997,2.377985,-0.167259,1.7389689999999998,1.640943,1.5445659999999999,-0.330843,-0.121575,2.749597,0.245202,2.9706490000000003,2.562383,-0.451345,1.34816,1.041007,0.618645,-0.15768,0.309824 -153,1,-0.8861450000000001,-1.527023,-0.923695,-0.7731,0.07589800000000001,-1.0468,-0.9644389999999999,-0.9066860000000001,-0.067552,-0.899167,-0.845593,-1.445027,-0.869073,-0.776409,0.083955,-1.008427,-0.8660329999999999,-0.801139,0.067109,-0.24774200000000002,-0.649918,-0.789881,-0.711388,-0.546898,0.659367,-0.921794,-0.6609430000000001,-0.578137,0.404124,-0.823039 -509,0,0.174117,1.734739,0.31092800000000004,0.050658,1.789881,1.542225,1.529986,1.54845,0.181587,1.260942,0.378508,1.084495,0.48732,0.21731999999999999,1.56135,1.566511,1.433786,0.9358280000000001,-0.016862000000000002,1.138675,-0.260123,1.3501379999999998,0.035118,-0.171989,0.832374,1.336982,0.9382209999999999,1.397252,0.061472000000000006,0.834081 -288,1,-0.9130659999999999,-0.545075,-0.8635280000000001,-0.778726,-1.296166,-0.445075,-0.564134,-0.326858,0.087755,-0.767831,-0.8143520000000001,0.155995,-0.751681,-0.741711,-1.1500540000000001,0.260754,0.049475,0.179548,2.860081,-0.066289,0.29389099999999996,1.248535,0.005416,-0.124465,2.899791,3.193632,1.627887,3.742804,1.651232,1.0296370000000001 -258,0,0.7415229999999999,0.971001,1.08536,0.607213,0.790423,1.672634,1.659519,2.003725,0.608681,0.9949459999999999,0.43531099999999995,0.9099649999999999,0.750937,0.33734200000000003,1.034726,3.924075,2.872586,2.290012,2.494987,2.586049,3.200748,2.244609,3.5909230000000005,2.159752,1.773079,1.9266159999999999,1.6719849999999998,2.991773,0.34237399999999996,0.849967 -228,1,-0.428493,0.9172629999999999,-0.494183,-0.45105100000000004,-0.423831,0.579746,0.570481,0.051676,-0.12093800000000002,0.65968,-0.42809200000000003,1.089149,-0.4374,-0.450759,-1.233317,-0.550559,-0.432134,-0.736654,-1.086589,-0.369656,-0.5784560000000001,-0.2737,-0.703468,-0.48023299999999997,-0.6240180000000001,-0.141855,-0.057485,-0.503683,-0.8345090000000001,-0.146725 -476,1,0.037443000000000004,0.257745,0.144127,-0.091558,-0.748217,0.563842,-0.10069299999999999,0.293779,-0.59333,-0.29735100000000003,0.02065,0.288638,0.018164,-0.103779,-0.501736,0.12240799999999999,-0.47921499999999995,-0.47303999999999996,-1.115796,-0.383832,-0.20706799999999997,-0.36078899999999997,-0.057948,-0.20521199999999998,-0.978033,0.41368699999999997,-0.12214100000000001,0.276383,-0.537867,-0.39750599999999997 -423,1,-0.233835,-0.028860000000000004,-0.174581,-0.30198,-0.774519,0.39717399999999997,0.450063,0.397319,-0.253596,0.24627800000000002,-0.132717,-0.037150999999999997,-0.103348,-0.22635999999999998,-0.41206800000000005,0.196319,0.097561,-0.020613,0.132826,-0.140004,-0.6524439999999999,-0.5839529999999999,-0.525752,-0.461532,-1.020368,0.145976,0.15903,0.153104,-0.57298,0.060935 -64,0,0.169975,1.269005,0.135192,0.013566,2.311528,0.965882,0.624693,0.867821,0.780165,1.061445,-0.411051,1.058897,-0.382205,-0.44336499999999995,1.127241,0.414261,0.30132800000000004,0.511001,0.34093,0.439796,0.07336799999999999,-0.070493,0.030168,-0.085302,0.24668400000000001,0.055994,-0.071742,0.179057,-0.5075970000000001,-0.073344 -137,1,-0.817808,-0.5955560000000001,-0.814083,-0.735833,-0.586024,-0.5691229999999999,-0.6346569999999999,-0.454456,-0.363605,-0.9030459999999999,-0.766069,-0.907474,-0.778867,-0.7255,0.0021149999999999997,-0.671849,-0.675072,-0.520243,-0.283381,-0.587967,-0.827491,-0.404696,-0.852967,-0.6087229999999999,-0.636019,-0.573882,-0.582029,-0.51812,-0.669842,-0.834764 -343,0,1.342063,1.462789,1.499383,1.159197,-0.46328400000000003,0.58229,0.965317,1.688535,1.8511360000000001,-0.264102,1.577048,0.556251,1.562384,1.533287,0.11455599999999999,0.560187,1.224118,1.583257,0.9871469999999999,-0.800607,0.7847430000000001,1.9361709999999999,1.1420059999999999,0.601151,-0.761691,0.45839799999999997,0.38317199999999996,1.016059,2.060478,-0.191737 -302,0,1.534649,0.611116,1.535126,1.433432,0.102199,0.539669,1.060308,1.1830120000000002,0.6361829999999999,0.595398,1.6934939999999998,1.063551,1.760097,1.6840240000000002,0.8283459999999999,1.505866,1.751427,2.0398099999999997,1.596855,1.68587,2.4067220000000002,0.9546110000000001,2.444432,1.9903389999999999,0.307686,1.220732,1.47868,1.226932,0.825477,0.806846 -561,1,-0.900641,2.055541,-0.955268,-0.7752100000000001,-1.7402229999999999,-1.267986,-1.305831,-1.745063,-2.1593419999999997,-1.379622,-0.831392,2.345765,-0.8773110000000001,-0.764748,-1.556408,-1.303122,-1.114873,-1.2618200000000002,-2.7441169999999997,-1.102557,-0.328698,4.860893,-0.40843,-0.385626,0.184348,-0.927774,-1.057501,-1.9134470000000001,-0.078979,-0.764788 -8,0,-0.161357,0.8228129999999999,-0.031609,-0.248363,1.662757,1.8183099999999999,1.280035,1.391616,2.389857,1.28865,-0.32016700000000003,0.58883,-0.18408,-0.384207,2.201839,1.6840099999999998,1.219096,1.150692,1.9656,1.572462,-0.35685,-0.389818,-0.22774299999999997,-0.352403,-0.43667700000000004,0.5332899999999999,0.12056800000000001,0.07524299999999999,0.107482,-0.017363 -502,1,-0.558955,-0.696519,-0.6133270000000001,-0.57762,1.1235760000000001,-0.5036,-0.39957800000000004,-0.464354,0.411311,-0.474682,-0.450813,-0.6910569999999999,-0.44151899999999994,-0.507926,1.383436,0.07882,-0.370614,-0.41603500000000004,0.479666,0.470983,-0.532258,-0.221084,-0.643569,-0.48067299999999996,0.887043,-0.548172,-0.203707,-0.030193,0.904178,-0.49736400000000003 -28,0,0.828498,1.796619,1.252161,0.6828029999999999,1.390974,2.269333,1.7334009999999997,1.3368,1.822016,0.82094,0.333066,1.3916680000000001,0.429654,0.22044899999999998,0.8425790000000001,1.23865,0.998129,0.9954120000000001,0.4176,0.36891599999999997,0.122092,-0.371675,0.31283,0.06959,-0.602684,0.284582,0.128194,-0.156717,-0.346563,-0.313156 -457,1,-0.397431,1.392767,-0.47571599999999997,-0.43540500000000004,-0.152049,-0.941264,-0.638975,-0.8307059999999999,-0.738931,-0.8903,-0.260524,1.387014,-0.324127,-0.332729,-0.601368,-0.990992,-0.766849,-0.7284,-0.70324,-0.9863139999999999,-0.710192,0.24157399999999998,-0.768317,-0.500695,-0.42434399999999994,-0.9722620000000001,-0.55683,-0.8670329999999999,-0.856303,-0.743984 -136,1,-0.6086550000000001,-0.032117,-0.628517,-0.586937,-0.230954,-0.9635290000000001,-0.80401,-0.684074,-1.923146,-0.582743,-0.686545,-0.60961,-0.710491,-0.65781,0.621966,-0.8223229999999999,-0.663898,-0.591176,-1.7255040000000001,-0.474559,0.157823,2.342584,0.194023,-0.13128599999999999,-0.154333,-0.652686,-0.174861,0.20663299999999998,-0.841774,-0.03325 -365,0,1.665111,0.11281400000000001,1.6066120000000002,1.5810959999999998,0.014527000000000002,-0.10601300000000001,-0.00954,0.9424319999999999,-0.471997,-0.9196709999999999,1.7928990000000002,0.579522,1.723026,1.8148529999999998,-0.345884,0.165996,0.115389,0.746242,-0.7068909999999999,-1.024589,0.6241329999999999,-0.5444,0.6692520000000001,0.7063189999999999,-0.277671,-0.358707,-0.270021,0.45643599999999995,-0.737646,-0.71448 -384,1,-0.42021000000000003,-1.3527799999999999,-0.317851,-0.451578,-0.69123,0.090554,0.06914,-0.348327,-0.26653899999999997,-0.595488,-0.240643,-1.296095,-0.25451599999999996,-0.321637,-0.905956,-0.352327,-0.47745699999999996,-0.52308,-0.710542,-0.972138,-0.800783,-1.244737,-0.630698,-0.551739,-0.923364,-0.265371,-0.11982000000000001,-0.539856,-0.718274,-0.447057 -81,1,-0.15307300000000001,-0.405029,-0.315766,-0.46704799999999996,0.9306979999999999,1.430264,1.0248059999999999,0.85564,1.013126,0.9783209999999999,-0.223602,-0.7981020000000001,-0.22568200000000002,-0.38363800000000003,0.814113,0.931636,0.35280300000000003,0.540406,0.47601499999999997,0.8820889999999999,-0.430117,-0.364417,-0.6589149999999999,-0.6023430000000001,-0.08233,0.57409,0.26214899999999997,0.329912,0.096584,0.305663 -306,1,-0.385006,-0.8512209999999999,-0.454568,-0.428374,-0.857807,-0.7612369999999999,-1.252098,-1.364398,-0.40405,-0.0053100000000000005,-0.263364,-0.8074100000000001,-0.325363,-0.334435,-0.8006310000000001,-0.982274,-1.09653,-1.177705,-0.655777,-0.546856,-0.775518,-1.1672639999999999,-0.8227700000000001,-0.546458,-1.136372,-0.984614,-1.0201,-1.5264149999999999,-0.859936,-0.45764799999999994 -282,0,1.557428,0.484098,1.3444969999999998,1.3138940000000001,0.851793,0.767407,0.764781,1.6839669999999998,1.115046,-0.336696,1.497524,-0.258223,1.451171,1.393926,0.522334,0.755387,0.926565,1.179323,0.297119,-0.549692,0.23722600000000002,-0.402337,0.018287,0.28212600000000004,-0.462345,-0.19495,-0.043227,0.518075,-0.522127,-0.692164 -475,1,-0.451272,-1.030349,-0.418229,-0.483045,0.010144,0.042844,0.36178899999999997,-0.255445,0.170262,-0.328384,-0.36844899999999997,-0.828354,-0.373967,-0.420896,-0.424166,-0.410318,-0.38229,-0.46788100000000005,-0.389259,-0.519922,-0.92133,-1.3232979999999999,-0.827225,-0.6313850000000001,-0.722689,-0.42018500000000003,0.042649,-0.557375,-0.548764,-0.5696100000000001 -118,0,0.8119310000000001,0.785359,0.68623,0.688077,2.329063,1.515507,2.2232279999999998,1.352027,0.603827,2.286134,0.469393,0.8424799999999999,0.565581,0.363223,1.362087,1.342883,1.563102,1.183192,1.03826,1.490241,0.529933,-0.262814,0.362333,0.402476,0.552696,0.798207,0.915342,0.341267,-0.677107,0.8280290000000001 -55,1,-0.710125,-0.5222760000000001,-0.7580859999999999,-0.658133,-0.327392,-1.062767,-0.8704069999999999,-0.783351,0.655596,-0.752869,-0.7405079999999999,-0.12558,-0.767334,-0.699334,-0.079725,-0.9402020000000001,-0.733704,-0.674233,0.395694,-0.5284270000000001,-0.289719,-0.467653,-0.33813499999999996,-0.371105,0.42902399999999996,-0.936493,-0.610212,-0.5070899999999999,1.409075,-0.53292 -557,1,-1.196769,1.394395,-1.214107,-0.966822,-1.098904,-1.162132,-1.305831,-1.745063,-0.688779,-0.789998,-1.336086,1.999032,-1.347292,-1.090967,-1.076753,-1.0353379999999999,-1.114873,-1.2618200000000002,-0.25417399999999996,-0.312952,0.477599,3.102794,0.372234,-0.247015,1.516402,-0.795763,-1.057501,-1.9134470000000001,1.149967,-0.17812 -284,1,-0.49061800000000005,-0.974982,-0.45099399999999995,-0.500975,-1.451345,-0.143545,0.29846100000000003,-0.196518,-1.4588430000000001,-0.702441,-0.351408,-0.835335,-0.32495100000000005,-0.393308,-1.293808,-0.161864,0.285006,-0.38740399999999997,-1.385966,-0.48873500000000003,-0.410988,0.31233299999999997,-0.25695,-0.375065,-0.207668,0.7898229999999999,1.570857,0.964152,-0.213376,-0.037411 -256,0,1.818352,1.724968,2.124886,1.837752,-0.187118,1.7725080000000002,0.733597,1.21042,-0.133881,0.917364,1.540126,2.206141,1.7147880000000002,1.57026,-0.267602,1.9322740000000003,1.124934,1.689012,0.297119,-0.06770599999999999,1.578769,-0.032392000000000004,2.1246419999999997,1.453497,-0.228336,1.239175,0.22335500000000003,0.550517,-0.143151,0.551149 -426,1,-0.8571540000000001,-0.668836,-0.77,-0.773804,0.014527000000000002,0.288394,0.10416199999999999,-0.327467,0.192911,0.693484,-1.0358829999999999,-1.002884,-1.0082959999999999,-0.9137790000000001,0.128078,-0.057630999999999995,-0.319515,-0.689709,0.41394899999999996,0.900517,-0.279974,-0.163025,-0.149529,-0.43050900000000003,0.107678,0.737287,0.687553,0.136883,0.129276,0.391527 -387,1,-0.15721500000000002,-0.929386,-0.226408,-0.23781599999999997,-2.0834580000000003,-0.8331209999999999,-0.782421,-1.054232,-0.580388,-0.981736,-0.070234,-0.72829,-0.14824500000000002,-0.16577999999999998,-1.8574380000000001,-1.06187,-0.858124,-1.04239,-0.747051,-1.14225,-0.545251,-1.079632,-0.5727800000000001,-0.378805,-1.104371,-0.633124,-0.398339,-0.775384,-0.493068,-0.690272 -497,1,-0.45748500000000003,-0.21775799999999998,-0.43014399999999997,-0.480408,-0.209036,-0.023315,-0.332893,-0.141702,0.217178,-0.40652,-0.47069399999999995,-0.46067700000000006,-0.474471,-0.497118,-0.503871,-0.5314180000000001,-0.661764,-0.65076,-1.042777,-0.331381,-0.9094200000000001,-0.790788,-0.7985140000000001,-0.625444,-1.081703,-0.657716,-0.611207,-0.762408,-0.6274649999999999,-0.852542 -298,1,-0.010186,-0.067942,-0.043523,-0.107027,-1.662195,-0.23896599999999998,-0.555018,-0.5985,-0.428317,-0.398208,0.037691,-0.26055,-0.030853,-0.061970000000000004,-2.177683,-0.9881489999999999,-0.804137,-0.90741,-0.644825,-0.9834780000000001,-0.632233,-0.9939950000000001,-0.596541,-0.43513,-1.290712,-0.654362,-0.6997359999999999,-1.0629819999999999,-1.151735,-0.69519 -157,1,0.403979,0.38964899999999997,0.388371,0.266178,-1.956334,-0.529682,-0.402937,-0.46054700000000004,-0.604655,-1.342493,0.7704479999999999,0.039642000000000004,0.676795,0.640807,-1.5592549999999998,-0.6085510000000001,-0.468292,-0.547068,0.11822300000000001,-1.434276,0.2661,1.529758,0.30342399999999997,0.138015,-1.199375,0.06325900000000001,-0.044222000000000004,-0.11292,-0.646838,-0.860864 -249,1,-0.749471,-0.730716,-0.785787,-0.683447,0.28631,-0.611108,-0.440357,-0.282091,-0.383019,-0.32450500000000004,-0.7405079999999999,-1.014519,-0.745503,-0.707013,0.351537,-0.49768500000000004,-0.571494,-0.506314,0.26060900000000004,-0.158433,-0.537671,-0.324502,-0.5841649999999999,-0.477813,-0.126331,-0.737638,-0.359545,-0.281618,-0.456744,-0.38237600000000005 -342,1,-0.900641,-0.940785,-0.8191470000000001,-0.774507,0.413434,-0.211612,-0.20288,-0.108204,0.6475069999999999,0.37983,-0.8711540000000001,-1.0075379999999998,-0.843535,-0.799162,0.49386800000000003,-0.2534,-0.43728100000000003,-0.400043,-0.13004200000000002,0.889177,-0.884517,-0.715493,-0.748021,-0.6498659999999999,0.12501199999999998,-0.374915,-0.14303,-0.27513000000000004,0.355692,-0.332068 -508,1,0.217604,-1.2892709999999998,0.07562,0.08370599999999999,0.132884,-0.751695,-0.371753,0.321186,-0.971891,-0.645362,0.6170800000000001,-0.835335,0.5243909999999999,0.469024,-0.148756,-0.705393,-0.42108500000000004,-0.08484,-0.36735300000000004,-0.8828280000000001,-0.716327,-1.35396,-0.851482,-0.432709,0.11767799999999999,-0.75161,-0.23852199999999998,0.407773,-0.859936,-0.4637 -267,1,-0.304244,0.710451,-0.28598,-0.385129,-1.396988,-0.5169590000000001,-0.60875,-0.802689,-0.735695,-0.7595189999999999,-0.152598,0.593484,-0.19808499999999998,-0.26703000000000005,-1.195599,-0.412213,-0.603634,-0.7093119999999999,-0.644825,-0.596472,-0.24243800000000001,1.268492,-0.136163,-0.298719,-0.535015,-0.07925900000000001,-0.029965,-0.30449,-0.029337000000000002,-0.297269 -338,1,-1.058024,0.189351,-1.05088,-0.87295,0.34329699999999996,-0.7256130000000001,-0.799692,-0.755486,-0.010929000000000001,-0.404858,-1.158009,-0.409482,-1.135162,-0.978625,0.308838,-0.5890310000000001,-0.799618,-0.803976,0.286166,0.07263700000000001,-0.517099,1.448112,-0.538623,-0.516756,0.254017,-0.6141220000000001,-0.497147,-0.608795,0.055417999999999995,-0.38464499999999996 -360,1,-0.5278930000000001,-0.764914,-0.608859,-0.518379,-1.7288259999999998,-1.342223,-1.288651,-1.496108,-1.080282,-1.592419,-0.450813,-0.28381999999999996,-0.516897,-0.46355799999999997,-1.56566,-1.4752020000000001,-1.099882,-1.121268,-1.035476,-1.5519379999999998,-0.195157,-0.47926499999999994,-0.265861,-0.264836,-0.419343,-1.161672,-1.03121,-1.326735,-0.013597,-1.0352379999999999 -12,0,0.971385,0.694167,1.323647,0.793551,-1.256713,0.8653719999999999,0.439988,0.945477,0.44528500000000004,1.017112,1.432201,1.2822959999999999,1.66536,1.3313549999999998,0.073992,2.680858,1.477729,1.621948,2.137194,2.155097,1.9862490000000002,4.265788,4.061202,1.669114,-1.300712,3.2131939999999997,1.890159,4.720928,2.941929,3.4213199999999997 -210,0,1.443533,0.352195,1.520233,1.3631149999999999,-0.6386270000000001,0.24004699999999998,0.546493,1.178444,0.013337,-1.4017879999999998,1.832661,0.663296,1.760097,1.8063209999999998,-0.388583,0.5772430000000001,0.944142,1.204343,-0.170202,-1.780172,1.64193,0.47743800000000003,2.0607830000000003,1.570106,0.36102199999999995,0.59421,0.762488,2.57165,1.358222,-0.540863 -533,0,1.441462,0.239833,1.332582,1.3437780000000001,-0.993698,-0.005503,0.177564,0.710987,0.516467,-1.1130719999999998,1.8014189999999999,0.32121700000000003,1.760097,1.8319169999999998,-0.341614,0.5109130000000001,0.797249,1.35627,1.293826,-1.220218,1.546286,0.94191,1.1395309999999998,1.321487,-0.701021,0.302466,0.299616,0.8424950000000001,0.9223399999999999,-0.406962 -167,0,0.78294,0.10141499999999999,0.698144,0.666982,-0.682463,-0.26950100000000005,-0.19376500000000002,0.499338,-0.146823,-0.646471,0.7534069999999999,-0.113944,0.713866,0.6581560000000001,-0.548706,-0.23729099999999997,-0.057492999999999995,0.43439300000000003,0.297119,-1.057194,0.699566,0.31596199999999997,0.6251939999999999,0.59411,-0.306005,-0.04349,-0.186797,0.688396,0.044521,-0.114196 -79,1,-0.42021000000000003,-0.139593,-0.458142,-0.454391,-0.152049,-0.255506,-0.475379,-0.538203,-0.196974,-0.264102,-0.359929,-0.30011,-0.36161,-0.42260299999999995,0.21205300000000002,-0.16830799999999999,-0.62661,-0.6646890000000001,-0.341796,-0.400843,-0.504106,-0.221084,-0.538623,-0.43975,-0.582683,-0.49563599999999997,-0.370819,-0.586735,-0.37077899999999997,-0.358168 -369,0,2.358838,0.019993,2.613373,2.366883,-0.130131,0.853922,0.975872,1.958046,-0.25845,0.099426,2.238801,0.607446,2.274975,2.352388,0.707364,1.725703,1.958584,2.609857,0.045204,-0.198126,2.175733,-0.937931,2.3241389999999997,1.977138,-1.021034,0.15268199999999998,0.128194,0.472657,-0.649259,0.00041500000000000006 -437,1,-0.126153,-0.667207,-0.180538,-0.229554,-0.564106,-0.821034,-0.769947,-0.610225,-0.28433400000000003,-0.643146,-0.024791999999999998,-0.770177,-0.090167,-0.12425599999999999,-0.838349,-0.860226,-0.67118,-0.502445,-0.3564,-0.541186,-0.057647000000000004,-0.309987,-0.109926,-0.167149,0.311686,-0.700192,-0.5243359999999999,-0.44609899999999997,-0.05960700000000001,-0.364976 -485,1,-0.515468,-0.756771,-0.28121399999999996,-0.5273439999999999,-0.651778,0.965882,1.043037,0.298347,0.534263,1.078069,-0.476375,-0.670114,-0.37561500000000003,-0.506788,-0.086842,0.8861530000000001,0.823615,-0.011843000000000001,0.9871469999999999,1.481735,-0.04718,-0.017878,1.058346,-0.223253,0.064343,2.751546,2.636192,2.500278,1.426026,2.205998 -408,0,0.996235,-0.043516,0.918559,0.8234360000000001,0.693984,0.758501,0.277832,1.2606680000000001,0.25762199999999996,0.060080999999999996,1.097064,0.31889,1.063983,0.957925,0.5152180000000001,0.493857,0.392979,1.014241,0.658562,-0.29877600000000004,0.175148,-0.623324,0.096502,0.20842,0.063343,0.125297,-0.22559099999999999,0.487256,-0.775181,-0.17358099999999999 -308,1,-0.26904,-1.422803,-0.35091300000000003,-0.319735,-1.8471830000000002,-1.24623,-1.239673,-1.408555,-1.0252780000000001,-1.220579,-0.17815999999999999,-1.531128,-0.258635,-0.252241,-1.608359,-1.29251,-1.0802459999999998,-1.147836,-1.630579,-1.339297,-0.6524439999999999,-0.962425,-0.671786,-0.43887,-1.234376,-1.214599,-0.990192,-1.387564,-0.6952689999999999,-0.8430860000000001 -414,0,0.205179,1.829188,0.08455599999999999,0.08933200000000001,-0.770135,-0.989865,-0.563654,-0.7439140000000001,0.5374979999999999,-1.235541,0.284783,2.448156,0.19528099999999998,0.18375999999999998,-0.936557,-1.1047,-0.526547,-0.555322,0.14743,-1.397419,0.22712,0.744147,0.087591,0.110953,-0.069996,-0.626418,-0.23222199999999998,-0.43944799999999995,1.3279530000000002,-0.7734880000000001 -41,0,-0.710125,1.573523,-0.5969439999999999,-0.644421,2.565776,0.098824,0.624213,0.423205,0.102315,0.671317,-0.9023959999999999,0.479458,-0.8266479999999999,-0.807125,1.8744779999999999,0.330874,0.195866,0.200441,0.304421,0.836725,-0.608412,0.383092,-0.516841,-0.514116,0.341021,-0.43806999999999996,-0.197076,-0.23133299999999998,-0.844196,-0.285543 -276,1,-0.842658,-1.088973,-0.890335,-0.7428640000000001,-0.283557,-1.150045,-1.282059,-1.575895,-0.23094699999999999,-1.1130719999999998,-0.794471,-1.193704,-0.8311780000000001,-0.7346010000000001,-0.18291500000000002,-1.243615,-1.0962040000000002,-1.175848,0.519826,-0.650341,-0.605164,0.11457,-0.6440640000000001,-0.511475,0.461692,-0.921067,-1.0081959999999999,-1.372803,0.3678,-0.8200129999999999 diff --git a/examples/data/breast_homo_test.csv b/examples/data/breast_homo_test.csv deleted file mode 100644 index aa997deb1..000000000 --- a/examples/data/breast_homo_test.csv +++ /dev/null @@ -1,115 +0,0 @@ -id,y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15,x16,x17,x18,x19,x20,x21,x22,x23,x24,x25,x26,x27,x28,x29 -146,0,-0.523751,0.114443,-0.456653,-0.507831,0.268776,0.9856030000000001,0.854974,1.094698,4.6482790000000005,1.055903,-0.660984,-0.6305529999999999,-0.534609,-0.633919,0.9066280000000001,1.244335,0.9679969999999999,0.650804,3.163109,1.546945,-0.308487,0.37946399999999997,-0.289622,-0.343602,-0.5380149999999999,0.606506,0.483969,1.076077,4.3270669999999996,0.317767 -405,1,-0.801242,-0.015832,-0.729789,-0.71755,0.172337,-0.571667,-0.62842,-0.544141,-1.0511620000000002,-0.367175,-0.905236,-0.162813,-0.8888450000000001,-0.810254,0.287488,-0.5636359999999999,-0.494155,-0.50554,-1.188815,0.475236,-0.092295,0.9546110000000001,0.075215,-0.320281,0.826041,-0.23127899999999998,-0.39767600000000003,-0.039925999999999996,0.029991000000000004,-0.413771 -35,0,0.774656,0.54435,0.7815439999999999,0.612486,1.0490549999999998,0.8221149999999999,1.28915,1.015519,3.174481,0.13212100000000002,0.742046,0.535307,0.746818,0.610375,-0.018522999999999998,0.554502,0.577537,0.290462,0.30807199999999996,-0.8842459999999999,0.203299,-0.539139,0.070265,0.106773,-0.421677,-0.027282,0.167651,0.024958,0.889649,-0.427388 -13,0,0.11820499999999999,0.322883,0.141149,-0.007178,-0.844656,-0.393548,-0.19184600000000002,-0.041207,-0.14844100000000002,-1.167934,0.489274,1.084495,0.48320100000000005,0.363507,-0.8789129999999999,-0.07847799999999999,0.13284,0.12177,0.129175,-1.3350440000000001,-0.006756999999999999,-0.25192800000000004,0.018287,-0.082662,0.909377,0.323146,0.617261,1.317769,1.122119,-0.299917 -166,1,-0.966908,-2.223994,-1.00084,-0.820212,0.492339,-0.8178529999999999,-0.80353,-1.044183,-1.311625,-0.38546199999999997,-0.944998,-2.2292490000000003,-0.955573,-0.845521,-0.029908999999999998,-0.890359,-0.797107,-0.823838,-1.572164,0.170451,-0.83868,-1.4704409999999999,-0.861383,-0.634905,0.256017,-0.87535,-0.693105,-1.046599,-0.9689059999999999,-0.6335350000000001 -173,1,-1.018678,-1.442344,-1.049987,-0.8509760000000001,-0.472051,-1.093302,-1.1169040000000001,-1.0894059999999999,-1.615768,-0.599367,-0.865474,-1.065715,-0.896259,-0.802575,0.301721,-0.889033,-0.818199,-0.595561,-0.89674,0.551787,-0.714162,1.0671,-0.7371300000000001,-0.467692,2.639781,-0.238544,-0.575729,0.654332,1.244409,0.37450500000000003 -531,1,-0.604513,0.510153,-0.6034970000000001,-0.580082,0.9920680000000001,0.26803699999999997,0.017325999999999998,-0.508663,0.49381800000000003,0.30779,-0.697906,0.169958,-0.690308,-0.678856,0.37288699999999997,-0.185933,-0.5875640000000001,-0.705443,0.172987,0.256925,-0.716327,-0.621147,-0.729209,-0.549978,-0.596684,-0.45874899999999996,-0.44741099999999995,-1.0540610000000001,-0.732803,-0.42511800000000005 -388,1,-0.875791,-1.098743,-0.82004,-0.7569279999999999,-0.97178,0.169436,0.143501,-0.485519,-1.203234,1.127944,-0.811511,-0.881877,-0.765686,-0.747684,-0.904533,0.133779,0.14941300000000002,-0.550679,-0.00591,1.37825,-0.269507,-0.271886,-0.147053,-0.38210500000000003,1.1130520000000002,2.303312,2.083133,1.406984,-1.011284,3.228412 -205,0,0.310792,-0.885418,0.31092800000000004,0.191466,0.733436,0.5015,0.29030500000000004,0.161307,0.8319340000000001,0.745575,0.281943,-0.607283,0.280545,0.175512,-0.540877,-0.160348,-0.166973,-0.20968299999999998,-0.794513,-0.416437,-0.48389399999999994,-1.5508170000000001,-0.441597,-0.305759,-0.523014,-0.351441,-0.381429,-0.573596,-0.643205,-0.34568499999999996 -6,0,1.368983,0.322883,1.368325,1.27522,0.51864,0.021215,0.509552,1.196716,0.26247600000000004,-0.014730000000000002,1.170908,0.16064900000000001,1.138125,1.0952950000000001,-0.123136,0.088295,0.300072,0.646935,-0.064325,-0.762332,0.149883,-0.80494,0.15541,0.298627,-0.9090299999999999,-0.651568,-0.310141,-0.228089,-0.829666,-0.611218 -144,1,-0.8944290000000001,-0.807253,-0.877825,-0.772397,-1.085753,-0.839482,-0.837832,-1.225379,-0.971891,-0.9008290000000001,-0.959199,-1.005211,-0.9765799999999999,-0.8520620000000001,-1.311599,-1.003499,-0.83226,-1.058692,-1.5064469999999999,-0.838883,-0.551026,0.040181,-0.524762,-0.497174,-0.164666,-0.428569,-0.388392,-1.003127,-0.464009,-0.542754 -540,1,-0.830233,-0.976611,-0.8483370000000001,-0.743216,0.093432,-0.270137,-0.443716,-0.6916869999999999,-0.924975,-0.144403,-0.7348279999999999,-1.1285459999999998,-0.713374,-0.716683,0.24763600000000002,0.14515,-0.269044,-0.592724,0.023298,0.711976,-0.45754700000000004,0.999969,-0.612877,-0.428529,1.7030759999999998,0.874216,0.783709,0.509965,-0.259386,0.6494939999999999 -30,0,1.424896,1.356941,1.5857620000000001,1.387726,0.733436,1.090566,1.636491,1.068812,0.87885,0.768849,1.2788329999999999,1.354435,1.352314,1.2318120000000001,0.714481,1.5987280000000001,1.796625,1.946952,1.355892,-0.11732200000000001,1.53582,0.452038,1.340514,1.422695,-0.264337,0.46175200000000005,0.665338,-0.03506,-0.057185,0.28939899999999996 -435,0,0.159621,0.8342120000000001,0.197742,-0.019835,1.268234,0.652266,0.646282,1.036837,0.45013800000000004,1.194443,-0.041832999999999995,0.076875,-0.034972,-0.157532,0.6860149999999999,0.169787,0.298817,0.40524499999999997,-0.5206930000000001,0.37458600000000003,-0.665437,-0.478177,-0.625748,-0.47275200000000006,-0.575683,-0.42298,-0.33302,-0.361588,-1.006441,-0.35892399999999997 -31,0,0.11406300000000001,0.397791,0.361564,0.014269,1.37344,2.0562259999999997,2.031327,0.608969,3.009467,3.117372,-0.649623,-0.137215,-0.578271,-0.60946,1.034726,0.8956280000000001,0.41432299999999994,0.074825,1.786704,2.153679,0.279093,-0.339016,0.301444,0.014584999999999999,-0.49668,0.48410699999999995,0.336752,-0.219979,0.264884,0.708123 -203,0,0.60692,2.633636,0.6326149999999999,0.47888500000000006,3.955374,1.6961709999999999,0.923099,1.320051,2.477217,1.366232,-0.090115,1.037954,-0.016848,-0.162367,2.5576659999999998,1.373206,0.841192,1.1050360000000001,1.604157,1.617825,0.5761310000000001,1.293893,0.516288,0.272445,0.594364,0.313085,-0.025654000000000003,0.180679,-0.068082,0.268973 -59,1,-1.400331,-1.673582,-1.410693,-1.064738,1.7942650000000002,-0.829304,-0.74548,-1.072199,0.516467,0.349906,-1.564717,-1.7452189999999999,-1.549947,-1.224071,0.08253200000000001,-0.978294,-0.856115,-1.0606520000000002,-0.46957899999999997,1.286106,-0.899675,-1.156197,-0.900985,-0.704386,1.0230489999999999,-0.834327,-0.40066,-0.9820399999999999,0.04331,-0.380863 -95,0,1.6464740000000002,0.962859,1.454704,1.528359,-0.586024,0.633818,0.660194,0.650081,1.2752059999999998,-0.014730000000000002,1.7417759999999998,0.8704049999999999,1.66536,1.732374,-0.397123,0.5109130000000001,0.72443,0.977872,1.0346090000000001,-0.8941690000000001,1.271986,0.530054,0.8355819999999999,1.045806,-0.341673,0.522112,0.34570500000000004,0.144993,0.729825,0.233039 -37,1,-0.614867,-0.466909,-0.679153,-0.588344,-1.549975,-1.323648,-1.073966,-0.9817530000000001,-1.478256,-1.233324,-0.311646,-0.202373,-0.3855,-0.37283099999999997,-0.46473000000000003,-1.263703,-0.7932140000000001,-0.507861,-1.258183,-0.590802,-0.798617,2.041403,-0.839601,-0.57594,-0.8963629999999999,-1.150159,-0.612202,-0.025327000000000002,0.746776,-0.763275 -331,1,-0.38293499999999997,-0.606955,-0.239812,-0.43294399999999994,-0.156432,0.45188199999999995,0.344038,-0.244025,1.1247530000000001,0.42748800000000003,-0.325847,0.014044,-0.306827,-0.40070300000000003,-0.040583999999999995,0.154625,-0.22259099999999998,-0.500897,-0.184806,0.36891599999999997,-0.49363900000000005,-1.180146,-0.198537,-0.43314899999999995,-0.438011,0.395243,0.39908699999999997,-0.322008,0.8436389999999999,0.29393800000000003 -237,0,1.6464740000000002,0.080246,1.621505,1.528359,-0.41944799999999993,-0.147362,0.209227,0.45518100000000006,-1.072193,-0.702441,1.804259,0.505055,1.6694790000000002,1.8518259999999998,-0.9116489999999999,-0.395347,0.020347999999999998,0.29149400000000003,-1.258183,-1.5632780000000002,1.01862,-0.31905900000000004,1.12765,0.949658,0.306019,0.327058,0.35399400000000003,0.797076,-0.863568,0.052235000000000004 -10,0,0.604849,1.3357709999999998,0.492622,0.47361099999999995,-0.6254770000000001,-0.6308279999999999,-0.6058720000000001,-0.22621,0.076431,0.031819,0.537556,0.919273,0.442011,0.406453,-1.017686,-0.713542,-0.700684,-0.404686,-1.035476,-0.826124,-0.09265599999999999,-0.054164,-0.198042,0.003805,-1.004034,-0.9059209999999999,-0.692442,-0.682114,-0.719485,-0.284787 -389,0,0.942393,0.7755890000000001,1.034724,0.760151,-0.318625,-0.08184,0.531141,1.033791,-0.525383,-0.439216,1.540126,0.912292,1.5211940000000002,1.476405,0.33018800000000004,0.520389,1.21533,1.371746,0.647609,-0.561032,0.741794,2.9376889999999998,1.245963,0.6548350000000001,1.39973,0.865833,1.4189969999999998,3.668187,0.8642219999999999,0.924861 -324,1,-0.52168,-0.6997760000000001,-0.48107700000000003,-0.522949,-0.296707,-0.391004,-0.7459600000000001,-0.899073,-0.387872,-0.24027300000000001,-0.5473779999999999,-0.949362,-0.574975,-0.560257,-0.685343,-0.737042,-0.864527,-0.825385,-0.633872,-0.213719,-0.532979,-0.743071,-0.44902200000000003,-0.469232,-0.5460149999999999,-0.631448,-0.70902,-1.079365,-0.8732540000000001,-0.654338 -179,1,-0.5465300000000001,-1.551449,-0.612433,-0.544747,-0.708765,-1.271103,-1.1735149999999999,-1.137522,-1.808284,-0.596042,-0.37413,-1.449681,-0.43987200000000004,-0.41549200000000003,-0.638374,-1.2621870000000002,-0.999455,-0.91876,-1.2618340000000001,-0.20804899999999998,-0.41965,-0.411772,-0.538623,-0.408067,0.49769399999999997,-1.06828,-0.85259,-0.7117979999999999,-1.197745,-0.167907 -220,1,-0.192419,-1.51888,-0.224919,-0.30655,-0.05561,-0.043670999999999995,-0.461946,-0.518408,-0.842468,0.179225,-0.13555799999999998,-1.42641,-0.168428,-0.244562,0.007097,-0.32655300000000004,-0.626736,-0.60072,-1.648834,0.091065,-0.703695,-1.4210909999999999,-0.7301989999999999,-0.504655,-0.969366,-0.476634,-0.5097470000000001,-0.8332930000000001,-0.8272450000000001,-0.40091 -50,1,-0.681134,0.006966,-0.723236,-0.640026,-1.046301,-1.069447,-1.0408629999999999,-1.179395,-0.756726,-1.0149860000000002,-0.6723439999999999,0.5376340000000001,-0.710491,-0.64558,-0.710963,-1.036286,-0.9068370000000001,-0.974217,-1.155957,-0.555362,0.00371,-0.012434,-0.114381,-0.261096,-0.394676,-0.878591,-0.670227,-0.705796,0.425918,-0.766301 -401,1,-0.5113260000000001,-0.901702,-0.584434,-0.511699,0.220556,-0.6155609999999999,-0.579486,-0.698082,-0.713046,-0.627629,-0.624062,-1.95,-0.652001,-0.603488,-0.543724,-0.983979,-0.78769,-0.79856,-0.768957,-1.0472709999999998,-0.552108,-0.311801,-0.602482,-0.470552,-0.288671,-0.751051,-0.60126,-0.967441,-1.16021,-0.733015 -368,0,2.998723,0.12421300000000002,2.74741,3.977131,0.172337,-0.581845,0.066741,1.026178,-0.632157,-1.053223,2.153597,-0.47463999999999995,2.015476,2.534411,-0.179357,-0.35479,0.351548,0.9216409999999999,-0.345447,-1.737644,2.893965,-0.300915,2.4092849999999997,4.0430980000000005,-0.49101300000000003,-0.80247,-0.362529,0.028201999999999998,-0.958009,-0.75344 -29,0,0.774656,-1.002666,0.8232440000000001,0.608971,-0.301091,0.171344,-0.11172699999999999,0.47193,-0.234183,-0.263547,0.977778,-0.9865950000000001,0.94865,0.8538309999999999,0.15013900000000002,0.21527,0.124931,0.7895760000000001,-0.265127,-0.185367,0.7042579999999999,-0.715493,0.8855799999999999,0.45682,-0.47134499999999996,0.27116799999999996,0.072159,0.282871,-0.15647,-0.020011 -78,0,1.4704540000000001,0.984029,1.877663,1.305104,1.382207,2.303684,2.379147,2.073768,4.10794,0.869706,1.719055,1.089149,2.130809,1.678336,2.2943540000000002,4.568425,3.5982629999999998,2.8755349999999997,3.9955239999999996,2.639918,1.90035,1.212248,2.862733,1.673514,1.1130520000000002,2.396089,2.559931,2.2926490000000004,7.071917,0.8291629999999999 -204,1,-0.26904,-0.168905,-0.333935,-0.356299,0.448503,-0.10474100000000001,-0.024412,-0.19956300000000002,0.183204,0.196958,-0.47069399999999995,-0.160486,-0.44811000000000006,-0.49199899999999996,0.234114,0.027651,-0.109847,-0.27623200000000003,0.41394899999999996,0.132176,-0.032743,-0.313616,-0.182696,-0.22105300000000003,-0.029327,-0.355912,-0.161929,-0.23133299999999998,-0.329612,-0.07901799999999999 -214,0,0.12234600000000001,1.49373,0.230506,-0.121794,1.03152,0.9646100000000001,0.49036199999999996,0.95309,2.949609,1.0337370000000001,0.01781,1.051916,0.037111,-0.12567799999999998,-0.123136,0.497647,0.285006,0.404988,1.545741,0.217232,0.056044000000000004,1.139674,0.33065100000000003,-0.205432,1.279726,0.64954,0.16533,0.725704,2.960091,0.368075 -127,0,1.253017,0.008594,1.219396,1.155681,-1.326851,-0.177261,0.232735,0.109537,-0.09667200000000001,-1.027177,1.383918,-0.08834700000000001,1.294648,1.374017,-1.009858,-0.455991,0.049099000000000004,0.189608,0.49061899999999997,-1.7518200000000002,1.0265600000000001,0.227059,1.1632930000000001,0.8997139999999999,-0.871028,0.102382,0.282043,0.29422600000000004,-0.029337000000000002,-0.419444 -46,1,-1.512777,-0.6053270000000001,-1.489328,-1.1222219999999998,-0.11698,-0.754239,-0.9757610000000001,-1.354653,0.330422,-0.546168,-1.6845709999999998,-0.57005,-1.658278,-1.288347,-0.737294,-0.8511299999999999,-0.9155,-1.109197,-0.15559800000000001,0.316465,-0.8982319999999999,-0.472008,-0.877224,-0.706961,0.642366,-0.50402,-0.530967,-0.9536530000000001,0.6293300000000001,-0.458783 -554,1,-0.49268900000000004,1.6386610000000001,-0.548691,-0.5008,-0.423831,-0.586935,-0.135715,-0.7564,-0.855411,-0.638713,-0.354249,2.241047,-0.390031,-0.39985,-1.076753,-0.873682,-0.337092,-0.657467,-0.89674,-0.810531,-0.698642,0.25971700000000003,-0.675251,-0.517196,0.45702600000000004,-0.22066,0.23496,-0.6774100000000001,-0.43495,-0.375945 -449,0,1.948814,1.041024,1.8151130000000002,2.006511,0.19425499999999998,0.355188,0.8046,1.7266009999999998,-1.02366,-0.5373020000000001,1.980348,0.28631100000000004,1.9001439999999998,2.073666,0.034138999999999996,0.249383,0.858769,1.717385,-0.940551,-0.877158,0.9352469999999999,0.261531,0.829642,0.914235,-0.5246810000000001,-0.264254,-0.00145,0.464546,-1.241333,-0.601383 -564,0,1.901185,0.1177,1.752563,2.015301,0.378365,-0.273318,0.664512,1.629151,-1.360158,-0.709091,2.110995,0.7214729999999999,2.0607860000000002,2.343856,1.041842,0.21906,1.947285,2.320965,-0.312589,-0.931027,2.78208,0.071025,2.3795830000000002,2.604187,1.086384,0.191805,0.6660010000000001,2.0671779999999997,-1.138416,0.16798 -297,0,-0.602442,-0.37246,-0.66009,-0.574808,-0.818354,-1.110223,-1.012222,-0.654838,-1.492816,-0.8199219999999999,-0.6723439999999999,-0.267531,-0.698958,-0.636479,0.236249,-0.856626,-0.777772,-0.355161,-0.70324,0.010262,0.8655889999999999,1.6114030000000001,0.62965,0.193019,-0.481679,-0.8622719999999999,-0.6357430000000001,0.409395,-0.580244,-0.39221 -491,1,0.7353109999999999,-1.181794,0.590915,0.579086,-1.4794,-0.9828680000000001,-0.80305,-0.475012,-1.808284,-1.398463,1.057302,-1.410121,0.932174,0.9590620000000001,-1.279575,-0.799203,-0.556804,-0.184147,-2.159966,-1.469717,0.282341,-0.309987,0.14699500000000001,0.233502,-0.8906959999999999,-0.9615309999999999,-0.6752,-0.707094,-0.910789,-0.9402959999999999 -492,0,1.089422,0.062333000000000006,1.076424,0.9587950000000001,-0.064377,-0.137184,-0.085341,0.5221779999999999,0.5666180000000001,-0.42646999999999996,1.102744,0.295619,1.088697,1.0014399999999999,0.266139,0.46543,0.354059,0.740309,1.1112790000000001,-0.287435,1.261881,0.129084,1.231112,1.086949,0.318687,0.08505599999999999,0.181577,0.759768,1.020413,0.076065 -181,0,2.155897,1.270634,2.062335,2.124291,0.733436,3.207003,1.9468900000000002,2.675218,1.936879,2.4634650000000002,1.9775080000000003,1.6941869999999999,2.089619,1.866047,1.2624549999999999,3.389643,2.007548,2.5969599999999997,2.129892,1.58522,0.810729,-0.823628,0.766278,0.904775,-0.929364,1.235822,0.22633899999999998,0.628378,-0.310239,0.5674130000000001 -427,1,-0.726692,1.036139,-0.702088,-0.6874899999999999,-0.090679,-0.538588,-0.381348,-0.605352,0.103933,-0.405966,-0.944998,0.626063,-0.954749,-0.8389790000000001,-0.594251,-0.889033,-0.661136,-0.8996719999999999,0.746185,-0.429195,-0.35179699999999997,0.7332609999999999,-0.30991799999999997,-0.44305,-0.166,-0.22345399999999999,-0.065774,-0.218356,-0.25454299999999996,-0.41793100000000005 -471,1,-0.552743,1.246207,-0.596349,-0.550197,-1.239179,-0.998771,-1.040815,-0.900443,-0.803642,-0.9728700000000001,-0.59282,2.059536,-0.6227560000000001,-0.58301,-0.629123,-0.840327,-0.817697,-0.6486970000000001,0.15473199999999998,-0.824707,0.725191,2.58752,0.6103430000000001,0.101712,0.15868,-0.554879,-0.57175,0.005493,-0.008754000000000001,0.033322000000000004 -515,1,-0.7867460000000001,-0.431084,-0.837316,-0.706651,0.698367,-0.616197,-0.526713,-0.444407,0.25762199999999996,-0.893071,-0.7916300000000001,-0.158159,-0.7912239999999999,-0.749959,0.607733,-0.36673,-0.574758,-0.592724,0.42125100000000004,-0.097476,-0.585313,-0.375303,-0.680696,-0.487274,0.5120279999999999,-0.506814,-0.361535,-0.11778599999999999,0.45982,-0.975096 -445,1,-0.681134,0.7625609999999999,-0.678558,-0.644597,-0.05561,-0.458434,-0.533909,0.085174,-0.488175,-0.079567,-0.607021,1.303239,-0.5914510000000001,-0.6074689999999999,0.47251899999999997,-0.230469,-0.43175699999999995,-0.159385,0.0306,0.808373,-0.515655,-0.023321,-0.495555,-0.460872,0.426358,-0.292198,-0.283616,0.787343,0.048152999999999994,-0.045732 -506,1,-0.643859,-0.245442,-0.659197,-0.6421359999999999,0.34329699999999996,-0.144817,0.390094,-0.513536,-0.310219,0.24627800000000002,-0.5416979999999999,0.174612,-0.514838,-0.573909,0.94221,0.205794,-0.088504,-0.703122,1.140487,0.870748,-0.808723,-0.763754,-0.9308350000000001,-0.610703,-0.256337,-0.195509,0.25750700000000004,-0.30854499999999996,-0.8889950000000001,0.006845 -93,1,-0.242119,0.042792000000000004,-0.288065,-0.318504,0.067131,-0.5036,-0.643293,-0.5404869999999999,-0.36036999999999997,-0.99282,-0.19236,-0.230298,-0.221152,-0.28381,0.41558599999999996,-0.43002799999999997,-0.615938,-0.544747,-0.633872,-0.8076949999999999,-0.397634,0.283304,-0.379718,-0.332602,-0.385675,-0.590648,-0.436801,-0.394192,-0.206112,-0.748145 -453,1,-0.097161,-1.424431,-0.123945,-0.22973000000000002,0.102199,-0.677266,-0.647131,-0.11734000000000001,-0.47685,-0.32395100000000004,0.114375,-1.2355909999999999,0.077889,-0.030400999999999997,0.9635600000000001,-0.22591999999999998,-0.24920799999999999,0.4135,-0.59006,-0.22505999999999998,-0.357933,-0.8991040000000001,-0.35793600000000003,-0.32204099999999997,-0.302672,-0.724783,-0.522015,-0.070746,0.184972,-0.08771799999999999 -357,1,-0.240048,-0.015832,-0.31338299999999997,-0.32729400000000003,-0.748217,-0.976252,-1.052282,-0.899073,-0.871589,-0.710199,-0.073075,-0.716655,-0.142066,-0.174028,-0.6355270000000001,-0.936601,-0.9262969999999999,-0.723241,-1.415174,-0.56245,-0.544529,0.26516,-0.5589189999999999,-0.43116899999999997,-0.467679,-0.9802540000000001,-0.883294,-0.933377,-0.617779,-0.646017 -544,1,-0.252473,-0.15099300000000002,-0.241004,-0.33749,-0.261639,-0.321664,-0.645212,-0.702802,-1.054398,0.053985000000000005,-0.073075,0.328198,-0.09057899999999999,-0.19934100000000002,-0.041296,-0.048155,-0.651846,-0.65076,-0.699589,0.578721,-0.480646,-0.308173,-0.391104,-0.378805,-0.24766999999999997,-0.21004099999999998,-0.190445,-0.443666,-0.683161,-0.074101 -26,0,0.279729,1.226666,0.450921,0.028683999999999998,0.8824780000000001,2.6083950000000002,1.351518,2.367641,2.20543,2.413591,0.128576,0.521345,0.224115,-0.028694,0.643316,1.56272,0.674211,1.003666,1.607807,0.913276,-0.5438069999999999,-0.42392799999999997,-0.374272,-0.424349,-0.863028,0.283464,-0.168561,0.279627,-0.726749,-0.031737 -356,1,-0.43056400000000006,-0.134708,-0.388443,-0.50871,0.084665,0.073378,-0.07142799999999999,0.170443,0.34336500000000003,-0.042993,-0.30596599999999996,-0.162813,-0.28334899999999996,-0.406391,0.8425790000000001,0.493857,0.090781,0.183417,0.815553,0.313629,-0.34133,0.5318689999999999,-0.136658,-0.412908,0.255351,0.7736149999999999,0.637486,1.1296059999999999,0.975614,0.726279 -227,1,0.029158999999999997,-1.0368629999999999,0.20667800000000003,-0.127243,-0.822738,0.6897979999999999,0.326287,0.354685,0.086138,-0.018055,0.247862,-0.87955,0.225762,0.084216,-0.9002629999999999,0.09966599999999999,-0.298172,-0.28680700000000003,0.253307,-0.5284270000000001,-0.625736,-1.3067870000000001,-0.292097,-0.45009099999999996,-0.974032,0.36841599999999997,0.150741,-0.039925999999999996,-0.803029,-0.22351 -377,1,-0.327023,1.6207479999999999,-0.30236199999999996,-0.351553,-0.9454790000000001,-0.690625,-0.9251950000000001,-0.8648129999999999,-0.334485,-0.739015,-0.18952,2.0758259999999997,-0.250397,-0.263902,-1.508016,-1.081769,-0.955299,-0.973701,-1.4261270000000001,-0.7325619999999999,-0.8527549999999999,-0.121295,-0.725744,-0.559439,-0.699688,-0.75161,-0.808558,-1.073364,-0.741279,-0.798452 -234,1,-1.192628,-1.061289,-1.236744,-0.9575049999999999,0.790423,-1.012194,-0.9622799999999999,-0.6453979999999999,-0.23256500000000002,-0.120021,-1.295188,-0.786467,-1.3081610000000001,-1.067361,-0.8340790000000001,-1.202869,-0.907465,-0.831834,-0.9515040000000001,0.174704,-0.685649,-0.701704,-0.817325,-0.609383,1.533069,-0.8427100000000001,-0.664258,-0.352504,0.39807,-0.09641799999999999 -341,1,-1.1429280000000002,-0.42457,-1.072624,-0.9263899999999999,-0.39753,0.5555720000000001,0.776774,-0.508663,0.131435,0.792678,-1.284111,-0.57005,-1.2492590000000001,-1.064801,-0.821981,-0.228573,-0.057492999999999995,-0.670622,0.819204,1.198214,-0.796813,-0.497046,-0.711388,-0.621924,-0.36234099999999997,0.515964,0.609634,-0.533043,-0.368357,0.089304 -129,0,1.317213,1.286918,1.234289,1.245335,-0.213419,0.8386549999999999,1.415804,0.892184,0.653979,0.039023,1.60829,1.356762,1.5829790000000001,1.527599,0.36577,1.033974,2.080367,1.7019090000000001,1.42526,-0.23640100000000003,0.325291,-0.032392000000000004,-0.050026999999999995,0.505884,-0.669353,0.352767,0.519116,-0.22160100000000002,-0.576612,-0.21594499999999997 -289,1,-0.809525,0.07536,-0.8331459999999999,-0.740579,-0.9016430000000001,-0.999916,-0.944625,-0.800557,0.592503,-0.776144,-0.7831100000000001,-0.093001,-0.815526,-0.7363069999999999,-0.656877,-1.028326,-0.813679,-0.7013159999999999,0.735232,-0.46038199999999996,-0.503745,1.373724,-0.45149700000000004,-0.502675,-0.167666,-0.644302,-0.60126,-0.303679,1.648811,-0.327908 -539,1,-1.572003,1.0117120000000002,-1.571835,-1.154919,1.193713,0.33165100000000003,0.321969,-0.983733,-0.179178,1.2554,-1.827998,1.4312280000000002,-1.797089,-1.3779370000000002,-0.688901,0.29486599999999996,0.046713,-0.90999,0.8228549999999999,2.085634,-0.669769,0.475624,-0.703468,-0.629405,2.809788,2.1848259999999997,2.0101869999999997,0.299092,0.061472000000000006,1.420748 -145,1,-0.64593,-1.492825,-0.6255390000000001,-0.6521560000000001,0.43973599999999996,-0.016316999999999998,-0.8539040000000001,-0.8250719999999999,-0.281099,1.089153,-0.632582,-1.079677,-0.570856,-0.631644,1.340737,0.478696,-0.649083,-0.48722600000000005,0.669515,2.210384,-0.032382,-1.021573,0.0767,-0.336782,1.043049,1.225762,-0.132751,-0.11292,1.298894,2.128835 -138,0,0.47231599999999996,-0.691634,0.42113599999999995,0.159648,0.382749,-0.240875,0.303738,0.793211,0.830317,-0.691358,0.233661,-0.400174,0.201048,0.066014,1.4474850000000001,0.495752,0.817337,0.9626530000000001,0.530779,-0.090388,3.215185,0.426637,2.7488759999999997,1.35449,0.9863799999999999,0.520995,1.123237,2.630045,3.7701059999999997,-0.343416 -410,1,-0.6666380000000001,1.73311,-0.660984,-0.631588,0.56686,-0.585662,-0.43699899999999997,-0.420653,0.116875,-0.359971,-0.78595,-0.400174,-0.802345,-0.7255,-0.553687,-0.9705239999999999,-0.765468,-0.720146,-0.768957,-0.519922,-0.770826,0.613514,-0.7460399999999999,-0.5869409999999999,-0.5500149999999999,-0.868028,-0.6718850000000001,-0.960952,-0.863568,-0.807909 -56,0,2.0440720000000003,0.401048,1.871706,2.222734,1.316453,0.616006,0.555129,1.438819,1.029304,-0.055738,1.443561,-0.167467,1.381147,1.413834,0.636199,0.42373599999999995,0.54615,1.058091,0.384741,-0.45187700000000003,1.163349,-0.043278,0.975676,1.367691,-0.19433399999999998,-0.135149,-0.081026,0.581337,-0.24485700000000002,-0.44894799999999996 -216,1,-0.625221,0.23169,-0.627326,-0.614185,0.35644699999999996,0.320201,0.040354,-0.012276,0.802814,-0.016393,-0.635423,-0.21866300000000002,-0.6033970000000001,-0.633351,-0.194302,0.209585,-0.281725,-0.45137299999999997,0.567289,0.048537000000000004,-0.392942,0.628029,-0.385658,-0.415328,0.6103649999999999,0.9189280000000001,0.9140159999999999,0.9819950000000001,0.894492,0.370723 -281,1,-0.612797,-1.207849,-0.672005,-0.6097899999999999,-1.261097,-1.076762,-0.9827170000000001,-0.482778,0.32395100000000004,-0.945716,-0.678025,-1.226283,-0.730262,-0.647287,-1.297366,-1.154922,-0.8330139999999999,-0.549132,1.0565149999999999,-0.23640100000000003,0.5656640000000001,0.092797,0.421242,-0.05516,0.33102,-0.618034,-0.555503,1.080943,1.049472,-0.677034 -16,0,0.5799989999999999,0.84724,0.480707,0.452516,0.615079,-0.42726400000000003,0.092168,0.704897,0.20747100000000002,-0.098963,0.156977,0.195555,0.11413699999999999,0.084216,0.164372,-0.612909,-0.186433,0.094686,-0.8237209999999999,-0.507163,0.243723,0.041996,0.16283599999999998,0.111393,-0.441011,-0.774525,-0.395023,-0.114542,-0.7800239999999999,-0.6467729999999999 -444,0,0.8512770000000001,-0.5955560000000001,0.775587,0.7232350000000001,-0.266022,0.078468,0.752307,0.59222,-0.095054,-0.093975,1.108425,-0.567723,1.051626,0.95309,-0.49035,0.357406,0.253619,0.35133600000000004,-0.334494,-0.7084630000000001,-0.384641,-1.136239,-0.467833,-0.100484,-0.974699,-0.552085,-0.071079,-0.331417,-0.919264,-0.514007 -547,1,-1.1263610000000002,-0.592299,-1.077688,-0.91971,0.601928,-0.188711,-0.45043199999999994,-0.47623000000000004,-0.339339,0.600939,-1.098366,-0.6305529999999999,-1.075848,-0.950184,-0.540166,-0.44878999999999997,-0.5677270000000001,-0.632962,-0.5206930000000001,0.615579,-1.0494569999999999,-0.351717,-0.92935,-0.7263,1.076384,0.299672,-0.191108,-0.134007,0.269727,0.792473 -270,1,-0.281464,-0.818652,-0.381891,-0.344521,-2.0470740000000003,-1.297121,-1.120358,-1.23756,-0.716282,-1.260478,0.046211,-0.574704,-0.068748,-0.06339199999999999,-2.282296,-1.470464,-1.023849,-1.100607,-1.108494,-1.281175,-0.992432,-0.901826,-1.001031,-0.650526,-1.183041,-1.216611,-0.8974850000000001,-1.328195,-0.6274649999999999,-0.913062 -44,0,-0.008116,0.686025,-0.052459000000000006,-0.245902,0.786039,0.866009,0.48268599999999995,0.701851,1.281677,0.677967,-0.271884,0.586503,-0.269756,-0.350931,0.055489,0.006804000000000001,-0.077958,0.09287999999999999,-0.23956999999999998,-0.145674,-0.762886,-1.096868,-0.758416,-0.56868,-1.230376,-0.65045,-0.5760609999999999,-0.801825,-1.139627,-0.7848350000000001 -34,0,0.8160729999999999,0.257745,0.757716,0.66874,0.536175,2.0746740000000004,1.224383,1.093175,2.215137,2.180844,0.5687979999999999,-0.32803499999999997,0.619129,0.433188,0.5436840000000001,0.9771200000000001,0.58507,0.73773,0.6804680000000001,0.333476,-0.256875,-0.9636950000000001,-0.33813499999999996,-0.11676500000000001,-0.952032,0.178951,-0.174198,-0.44269200000000003,-0.42526400000000003,0.008357999999999999 -371,1,-0.014328,-1.619844,-0.082245,-0.108082,-0.866574,-0.512506,-0.652408,-0.499832,-0.669366,-0.902492,0.301824,-1.414775,0.234,0.16186,-1.190618,-0.66332,-0.688883,-0.5764729999999999,-0.330843,-1.043018,-0.818829,-1.459374,-0.756436,-0.49761400000000006,-0.676354,-0.594002,-0.543235,-0.428418,-0.493068,-0.767057 -310,1,-0.757754,0.142126,-0.7845949999999999,-0.698741,-0.441366,-0.925997,-0.926107,-0.8709040000000001,0.948415,-0.796094,-0.689385,-0.041805,-0.7265550000000001,-0.671746,-0.585,-0.9818950000000001,-0.916128,-0.9657049999999999,0.45410900000000004,-0.215137,-0.884517,0.386721,-0.869798,-0.639305,-0.325673,-0.914808,-0.712005,-0.6744899999999999,0.356903,-0.8071520000000001 -538,1,-1.489377,0.8537540000000001,-1.492009,-1.112026,-0.296707,-1.08694,-1.305831,-1.745063,0.254387,0.855298,-1.817206,1.442863,-1.8119169999999998,-1.3540459999999999,-1.094545,-1.0529629999999999,-1.114873,-1.2618200000000002,0.21314699999999998,1.425031,-0.099152,0.44478,-0.185171,-0.466372,1.8730830000000003,-0.88228,-1.057501,-1.9134470000000001,1.0022520000000001,1.163916 -72,0,1.4601,1.326001,1.320668,1.407063,1.145493,3.086136,1.844223,1.146468,0.666921,2.768252,0.872693,1.2171379999999998,0.9156979999999999,0.780737,0.764297,1.490705,1.009428,0.787254,0.42125100000000004,0.293783,0.669609,-0.31905900000000004,0.41530100000000003,0.6409739999999999,-0.407009,1.714795,0.35233600000000004,-0.085345,-0.638362,0.9471780000000001 -73,0,0.062292999999999994,-0.784455,0.090513,-0.11986,0.382749,0.635726,0.027401,0.36077600000000004,-0.504352,1.055903,-0.092956,-0.814392,-0.06339299999999999,-0.20133099999999998,0.308838,0.44837299999999997,-0.136966,0.045676999999999995,-0.546249,0.40577399999999997,-0.45646400000000004,-1.08199,-0.45001199999999997,-0.373745,-0.774691,-0.269842,-0.47426899999999994,-0.420145,-1.010073,-0.25150100000000003 -279,1,-0.266969,-0.641152,-0.264832,-0.370187,-0.607942,-0.520776,-0.6073109999999999,-0.223469,0.087755,-0.821585,-0.07875499999999999,-0.956343,-0.122707,-0.191946,-0.08541900000000001,-0.520426,-0.5525359999999999,-0.304605,1.089373,-0.604978,-0.567628,-0.539501,-0.512881,-0.46043199999999995,-0.935364,-0.562704,-0.574403,-0.22160100000000002,-0.637151,-0.832873 -196,0,0.025018000000000002,1.356941,0.129234,-0.129529,1.8117990000000002,0.36854699999999996,0.521546,0.802347,0.289978,0.5200319999999999,-0.101476,0.698202,-0.055154999999999996,-0.18767999999999999,1.682331,0.42373599999999995,0.623991,0.421496,0.081713,0.846648,0.772111,1.624104,1.009833,0.206,2.253098,0.44722,0.489274,1.428072,0.76857,0.193323 -74,1,-0.44713,-0.40177199999999996,-0.522778,-0.473728,-0.6473939999999999,-0.445075,-0.48881199999999997,-0.426439,-0.457437,-0.435337,-0.516137,-0.644516,-0.526371,-0.523284,-0.330228,-0.6832189999999999,-0.691519,-0.67578,-0.334494,-0.518504,-0.558244,-0.348088,-0.557434,-0.454491,-0.729023,-0.407331,-0.452053,-0.621447,-0.809083,-0.563936 -23,0,2.671532,1.614234,2.404872,3.048953,0.338913,0.036482,0.207788,1.3139610000000002,-0.127409,-0.481332,1.9973889999999999,0.872732,1.863073,2.130548,-0.148044,-0.040575,0.262407,0.964717,-0.15559800000000001,-1.4201,1.03414,-0.163025,0.71133,1.180456,-0.7710239999999999,-0.720312,-0.488858,-0.22971100000000003,-1.175951,-0.6838420000000001 -334,1,-0.604513,0.453158,-0.677068,-0.591156,-0.44574899999999995,-1.041647,-1.132208,-1.138587,-0.560975,-0.658108,-0.518977,-0.062749,-0.58033,-0.541771,-0.9415389999999999,-1.181075,-1.017496,-1.041668,-0.995315,-0.474559,-0.7982560000000001,0.5717840000000001,-0.8252450000000001,-0.596182,0.280018,-0.952701,-0.825269,-0.8555159999999999,-0.139519,-0.594953 -400,0,0.9382520000000001,0.34242399999999995,1.261096,0.74433,2.4079669999999997,2.146558,3.028252,1.245441,0.556912,1.986889,1.074343,0.40266399999999997,1.3358379999999999,0.9644659999999999,1.8958270000000002,2.9044849999999998,2.888907,1.8282990000000001,1.100326,1.184038,-0.007839,-0.802218,0.127194,0.025806,0.039342,0.654012,0.9866299999999999,-0.20862399999999998,-0.560872,0.49327600000000005 -431,1,-0.701842,-0.450625,-0.525756,-0.641257,0.553709,0.05492999999999999,-0.15298599999999998,-0.622863,-0.557739,0.5344399999999999,-0.490575,-0.374576,-0.432457,-0.532101,0.643316,0.516599,-0.142993,-0.539846,-0.002259,1.1656090000000001,-0.8246040000000001,0.44115200000000004,-0.32774000000000003,-0.547998,0.9863799999999999,0.417599,0.554262,-0.020461,0.16075599999999998,0.8359719999999999 -254,0,1.9529560000000001,-0.180304,1.6632049999999998,1.9186159999999999,0.759738,0.393357,0.76526,1.298734,0.773694,0.30779,1.511725,0.00939,1.4223370000000002,1.462184,0.508101,0.27402,0.616458,0.954141,-0.13004200000000002,-0.8970049999999999,0.688377,-1.057134,0.460844,0.674637,-0.797359,-0.41795,-0.146014,0.141749,-0.8345090000000001,-0.470887 -48,1,-0.519609,-0.8105100000000001,-0.517714,-0.523828,0.746587,-0.245964,0.157414,-0.748025,-0.248743,-0.051859,-0.5899800000000001,-1.084331,-0.5737399999999999,-0.584717,0.47963500000000003,-0.254348,-0.287249,-0.552743,-0.498787,-0.335633,-0.5109630000000001,-0.884408,-0.50397,-0.45031099999999996,-0.5176810000000001,-0.626418,-0.287595,-0.995341,-0.7582300000000001,-0.519303 -130,1,-0.606584,-1.281128,-0.47303500000000004,-0.5895739999999999,0.45288599999999996,0.02694,-0.8301559999999999,-0.498461,0.9192950000000001,0.46905,-0.550218,-1.396158,-0.5309020000000001,-0.56623,0.7287140000000001,-0.17532,-0.756428,-0.518437,0.24965700000000002,0.27110100000000004,-0.738704,-0.726742,-0.442092,-0.552179,-0.08932999999999999,-0.051314,-0.719962,-0.600522,0.735879,0.131668 -555,1,-1.12429,1.5035,-1.1226639999999999,-0.919359,0.264392,-0.529682,-0.346326,-0.355331,-1.091607,-0.061834,-1.0898450000000002,1.936202,-1.0832620000000002,-0.948477,-0.43128299999999997,-0.526112,-0.3617,-0.55558,-0.798164,-0.21655500000000003,-0.668686,1.854525,-0.707428,-0.56934,1.669741,0.105176,0.535363,0.878181,-0.255754,0.432378 -520,1,-1.1802030000000001,-1.276243,-1.174194,-0.9738540000000001,2.3071450000000002,-0.283496,-0.8262700000000001,-0.639307,1.262264,0.325523,-1.372439,-1.254207,-1.318458,-1.129362,2.89926,0.34414,-0.696541,-0.637347,1.407006,2.007666,-0.18541300000000002,-0.157582,-0.236654,-0.45559099999999997,2.806454,-0.004367,-0.329041,0.649465,2.352277,0.040131 -122,0,2.019222,-0.274754,2.193393,2.096165,1.632072,1.082296,1.478172,1.677876,0.519703,-0.213673,2.874993,0.21184499999999998,3.057588,3.145893,3.440117,3.455973,4.243589,3.9279300000000004,3.079138,0.846648,3.983947,3.4529620000000003,3.435978,4.238914,5.429894,4.056567,3.179967,1.0420129999999999,3.018209,2.299805 -496,1,-0.391218,-0.574386,-0.35657300000000003,-0.43399899999999997,0.9175469999999999,0.8265680000000001,0.412642,0.397319,0.532645,1.0725280000000001,-0.419572,-0.26055,-0.382205,-0.481476,0.79988,0.550711,-0.108341,0.046967,-0.622919,0.814044,-0.623571,-1.058948,-0.5792149999999999,-0.482653,-0.44567799999999996,-0.025605000000000003,-0.183482,-0.23944400000000002,-0.35746,-0.08771799999999999 -225,1,0.103709,-1.429316,0.09349099999999999,-0.012979,-0.11698,-0.647368,-0.522875,-0.08993200000000001,0.260858,-1.2870780000000002,0.060412,-1.354271,0.022283,-0.038932999999999995,0.192127,-0.532555,-0.396226,-0.074523,0.96159,-1.179108,0.42165699999999995,-0.734362,0.444013,0.174978,0.016008,-0.625859,-0.274995,0.172569,0.25640799999999997,-0.8820459999999999 -326,1,-0.15307300000000001,-1.2501879999999999,-0.263939,-0.22973000000000002,-0.187118,-0.9120020000000001,-1.051226,-0.848216,-1.295447,-0.726824,-0.004911,-1.491568,-0.07986900000000001,-0.10918299999999999,-0.232731,-0.971851,-0.893278,-0.55687,-1.601372,-0.8218719999999999,-0.534423,-0.24648499999999998,-0.647529,-0.36120399999999997,-0.11633099999999999,-0.791292,-0.867942,-0.838484,-0.772759,-0.49887700000000007 -106,1,-0.648001,0.583433,-0.647878,-0.630885,1.5970030000000002,0.074651,0.072498,0.109537,-0.153294,0.389251,-0.706426,-0.22331700000000002,-0.691956,-0.689379,1.269571,-0.050051,-0.22723600000000002,-0.36289899999999997,-0.038768000000000004,0.340564,-0.357933,0.7985770000000001,-0.35199600000000003,-0.433809,0.49969399999999997,-0.132913,-0.081026,0.354244,-0.592352,0.017058 -381,1,-0.8654370000000001,-0.7893399999999999,-0.82004,-0.762026,-1.002465,-0.356652,-0.560775,-0.716658,0.48734700000000003,-0.613775,-0.876835,-1.014519,-0.8773110000000001,-0.802575,-1.173538,-0.635841,-0.669674,-0.726852,0.698723,-0.04786,-0.869719,-0.337202,-0.7846529999999999,-0.630505,-0.581683,-0.36038400000000004,-0.486205,-0.784468,-0.21095500000000003,-0.701242 -415,1,-0.6666380000000001,0.24960300000000002,-0.660388,-0.628776,0.448503,-0.22624299999999997,-0.7473989999999999,-0.486889,0.281889,-0.578309,-0.635423,0.43756999999999996,-0.6417039999999999,-0.6288,0.097477,-0.43855600000000006,-0.794093,-0.699769,0.757138,0.014515,-0.470901,-0.025135,-0.46337799999999996,-0.457791,0.9513790000000001,0.282346,-0.517041,-0.40879099999999996,0.24672199999999997,-0.57604 -71,1,-1.353531,-1.629614,-1.3314629999999998,-1.048038,-0.5115029999999999,-0.067845,-0.617866,-1.016318,-1.046309,1.355149,-1.488033,-1.0820040000000002,-1.366651,-1.168611,0.104593,0.924055,-0.034392,-0.521016,0.329977,3.82787,0.43681499999999995,-0.661607,0.14947,-0.327761,3.389811,3.811771,0.8228340000000001,0.951175,0.589374,6.859624 -558,1,-0.163427,0.259374,-0.040545,-0.25855900000000004,-1.304933,0.39971799999999996,0.45102200000000003,-0.062524,-1.0398379999999998,-0.216444,0.131416,0.788958,0.1821,0.006288,-0.827674,0.543131,0.177034,-0.298156,-1.3056450000000002,-0.188203,-0.648835,-0.197498,-0.317839,-0.457571,-0.9330309999999999,1.168754,1.1235680000000001,0.6916399999999999,-0.503965,0.231148 -21,1,-1.2506110000000001,-1.631243,-1.254913,-0.9944219999999999,0.001377,-0.887193,-0.880434,-0.796903,-0.729224,-0.344455,-1.31308,-1.593959,-1.302806,-1.083572,0.42981899999999995,-0.7470859999999999,-0.743748,-0.726337,0.012345,0.8863409999999999,-0.461517,-0.435539,-0.473774,-0.542058,0.855042,-0.6236229999999999,-0.399334,0.391552,-0.03297,-0.31277699999999997 -316,1,-0.708054,-1.499339,-0.7643409999999999,-0.646003,-1.414523,-1.278291,-1.108365,-1.463066,-0.983215,-1.306473,-0.5530579999999999,-1.21232,-0.60628,-0.550303,-1.353587,-1.368694,-0.973881,-1.131534,-0.506089,-0.8941690000000001,-0.699725,-1.11991,-0.706933,-0.5394180000000001,-0.566016,-1.101646,-0.6851470000000001,-1.094126,-0.09350900000000001,-1.0760129999999999 -463,1,-0.724621,-0.269869,-0.732172,-0.677646,0.08028099999999999,-0.46734,-0.413491,-0.48308199999999996,0.500289,-0.29458,-0.717787,-0.216335,-0.745091,-0.688811,-0.802766,-0.867807,-0.6921470000000001,-0.803461,-1.079287,-0.596472,-0.8069189999999999,-0.818729,-0.773762,-0.603883,-0.11066400000000001,-0.47328000000000003,-0.36783499999999997,-0.697362,0.084477,-0.388428 -454,1,-0.399502,-0.574386,-0.465887,-0.43435100000000004,-0.43259899999999996,-0.652457,-0.400538,-0.24509099999999998,0.597356,-0.589947,-0.42809200000000003,-0.497911,-0.46746899999999997,-0.460714,-0.7493920000000001,-0.948351,-0.742492,-0.67578,-0.04607,-0.643253,-0.8516729999999999,-0.9968969999999999,-0.866333,-0.594422,-1.051036,-0.9467209999999999,-0.641048,-0.796147,-0.540288,-0.815852 -413,1,0.101638,0.9563450000000001,0.087534,-0.023702,-1.0813700000000002,0.510406,0.188118,0.24657600000000002,0.424254,0.474591,0.24502100000000002,0.656315,0.229057,0.110382,-0.7977850000000001,-0.034888999999999996,-0.253727,-0.262045,0.483317,-0.519922,-0.31245700000000004,0.216173,-0.275266,-0.260216,-0.864028,0.14541700000000002,0.04066,0.02658,-0.179475,0.08325199999999999 -330,0,0.515803,-0.60207,0.5075149999999999,0.332978,0.487955,1.2311530000000002,1.0718219999999998,1.271326,0.19129300000000002,0.404213,0.540396,-0.87955,0.5697,0.39337,-0.10321,0.6208319999999999,0.396746,0.554335,-0.108136,-0.430613,-0.24568600000000002,-0.8513870000000001,-0.117352,-0.15548800000000002,-0.40067600000000003,0.389654,0.174283,0.453192,-0.711009,-0.254905 -468,0,1.097705,0.5199239999999999,1.082381,0.978132,-0.5115029999999999,1.426448,1.1720899999999999,1.294166,-0.9702729999999999,2.13097,0.986298,0.940217,1.1134110000000002,0.9260709999999999,-0.246964,1.8204599999999997,1.566869,1.322737,-0.422117,1.54411,1.8902439999999998,0.45022299999999993,1.4528860000000001,1.420495,-0.091663,2.502279,1.127879,1.835218,-0.46158699999999997,2.838813 -322,1,-0.46162600000000004,-0.748629,-0.43073900000000004,-0.49412,0.978917,-0.19825299999999998,-0.44659399999999994,0.013609,-0.839233,0.08778899999999999,-0.359929,-1.389177,-0.37685100000000005,-0.42686899999999994,1.212639,-0.303242,-0.637784,-0.384824,-0.9807110000000001,0.27818899999999996,-0.663994,-0.317244,-0.619808,-0.5229159999999999,-0.377008,-0.29722800000000005,-0.426854,-0.273508,-1.031867,-0.260201 -307,1,-1.360572,-0.913101,-1.380908,-1.0461040000000001,-1.479838,-1.284653,-1.235211,-1.533565,0.145995,-0.327276,-1.456224,-1.1378540000000001,-1.4667430000000001,-1.162069,-1.8723830000000001,-1.386888,-1.0686579999999999,-1.1722629999999998,-0.08623,0.784274,-0.832183,0.159928,-0.8524719999999999,-0.67211,0.11601199999999999,-1.151053,-0.93545,-1.350255,0.7831,-0.621052 -373,0,1.8846189999999998,-0.40828600000000004,1.7734130000000001,1.8729099999999999,1.044671,0.32592600000000005,0.689459,1.470795,-0.342574,-0.74234,1.849702,-0.45136899999999996,1.764216,1.934305,-0.135234,0.061763,0.802271,1.0444200000000001,-0.8784850000000001,-1.13658,0.752621,-1.014859,0.6202439999999999,0.807087,-0.276671,-0.36485500000000004,-0.168561,0.084976,-0.942269,-0.788239 -304,1,-0.743259,-0.6623220000000001,-0.731874,-0.686963,-0.787669,-0.47942700000000005,-0.717654,-0.90623,-1.120727,-0.41926599999999997,-0.757549,-0.26287699999999997,-0.7570359999999999,-0.716114,-0.557245,-0.519289,-0.6950350000000001,-0.874394,-1.462636,-0.052113,-0.279252,-0.2864,-0.19358599999999998,-0.38298499999999996,-0.129665,0.058228999999999996,-0.321083,-0.647077,-0.194004,-0.14521199999999998 -529,1,-0.583805,-1.61333,-0.60588,-0.5813119999999999,0.864944,-0.5793010000000001,-0.527672,-0.61936,-0.19373800000000002,-0.189844,-0.5843,-1.3612520000000001,-0.58239,-0.596377,0.970677,-0.270077,-0.640169,-0.540104,-0.564504,0.46531300000000003,-0.555357,-1.293361,-0.5703050000000001,-0.47957299999999997,0.09534400000000001,-0.779555,-0.461337,-0.618041,-0.111671,-0.590414 -40,0,-0.07024,0.744648,-0.141817,-0.162929,-1.0068489999999999,-0.317847,-0.30554699999999996,-0.051864999999999994,0.150849,-0.691912,-0.19520099999999999,0.53298,-0.23845100000000002,-0.261342,-1.048999,-0.8344520000000001,-0.724413,-0.7379439999999999,-0.10083400000000001,-0.9820610000000001,-0.601554,-0.7082350000000001,-0.640599,-0.43578999999999996,-1.25371,-0.808059,-0.596618,-0.797283,-0.816347,-0.948996 -115,1,-0.538247,0.076989,-0.587413,-0.523125,0.772888,-0.091382,-0.584763,-0.641591,-0.748637,0.081139,-0.624062,0.521345,-0.635937,-0.6151479999999999,0.093918,-0.489914,-0.6970430000000001,-0.743876,-0.451325,-0.121575,-0.336999,-0.533695,-0.428726,-0.342062,0.254017,-0.022811,-0.44906899999999994,-0.6626489999999999,-0.9398479999999999,0.02311 -2,0,1.51187,-0.023974000000000002,1.347475,1.456285,0.527407,1.082932,0.854974,1.955,1.152255,0.20139100000000001,1.579888,0.456187,1.566503,1.558884,0.94221,1.052926,1.363478,2.0372310000000002,0.939685,-0.398008,1.228676,-0.780083,0.850928,1.1813360000000002,-0.297005,0.814974,0.21307600000000002,1.424827,0.23703600000000002,0.293559 -39,0,-0.15307300000000001,0.05581900000000001,0.001155,-0.24643,1.255083,1.070209,1.107324,1.6931029999999998,-0.151676,1.283108,-0.18384,0.356123,-0.147009,-0.27215,0.37288699999999997,0.400995,0.219721,0.14111500000000002,-0.334494,0.19738599999999998,-0.693589,-1.134788,-0.653965,-0.480013,-0.558016,-0.172595,-0.046543,0.133639,-0.8199799999999999,-0.22994 diff --git a/examples/download/download_table.json b/examples/download/download_table.json deleted file mode 100644 index 5c80a5e75..000000000 --- a/examples/download/download_table.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "output_path": "./experiment_download_breast_guest.csv", - "namespace": "experiment", - "table_name": "breast_hetero_guest" -} diff --git a/examples/intersect/test_rsa_job_conf.json b/examples/intersect/test_rsa_job_conf.json deleted file mode 100644 index af21a37fb..000000000 --- a/examples/intersect/test_rsa_job_conf.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "initiator": { - "role": "host", - "party_id": 10001 - }, - "role": { - "host": [ - 10001 - ] - }, - "role_parameters": { - "host": { - "args": { - "data": { - "id_library": [ - { - "name": "aa60b8a29be511e9a24e00e04c6c66f7", - "namespace": "host-10000#id_library_input" - } - ] - } - } - } - }, - "algorithm_parameters": { - "rsa_0": { - "rsa_key_n": 127723942009318209263400084778770628480830638669639474401827682322234257084189804839006751667509865995109594304364220116526581975420126322508339108237805249456994757963061040563295555807508554542250179528965186893786470238549773710028212909393615132828050640874733786667120435977078222500555817644890641892379, - "rsa_key_e": 65537, - "rsa_key_d": 56727110518275627143970855664709493393133311184163617516077929462661300304836271790765354290889778666290073030719769550662945890863807115559324970520956339740859933640441993807668793671361749788702861745739816295257046496980424568438831350465242668254824453485420303929956114164660667388201518759657460036673, - "save_out_table_namespace": "host-10000#id_library_input#rsa", - "save_out_table_name": "aa60b8a29be511e9a24e00e04c6c66f7" - } - } -} diff --git a/examples/intersect/test_rsa_job_dsl.json b/examples/intersect/test_rsa_job_dsl.json deleted file mode 100644 index 025a7523a..000000000 --- a/examples/intersect/test_rsa_job_dsl.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "components": { - "rsa_0": { - "module": "Rsa", - "input": { - "data": { - "data": [ - "args.id_library" - ] - } - } - } - } -} diff --git a/examples/key/save_public_key.json b/examples/key/save_public_key.json deleted file mode 100644 index 7062eb839..000000000 --- a/examples/key/save_public_key.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "party_id": 9999, - "key": "xxx" -} \ No newline at end of file diff --git a/examples/lr/test_hetero_lr_job_conf.json b/examples/lr/test_hetero_lr_job_conf.json deleted file mode 100644 index b1d97a3c7..000000000 --- a/examples/lr/test_hetero_lr_job_conf.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "dsl_version": "2", - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "guest": [ - 9999 - ], - "host": [ - 10000 - ], - "arbiter": [ - 10000 - ] - }, - "job_parameters": { - "common": { - "task_parallelism": 2, - "computing_partitions": 8, - "task_cores": 4, - "auto_retries": 1 - } - }, - "component_parameters": { - "common": { - "intersection_0": { - "intersect_method": "raw", - "sync_intersect_ids": true, - "only_output_key": false - }, - "hetero_lr_0": { - "penalty": "L2", - "optimizer": "rmsprop", - "alpha": 0.01, - "max_iter": 3, - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - } - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": true, - "label_name": "y", - "label_type": "int", - "output_format": "dense" - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": false, - "output_format": "dense" - }, - "evaluation_0": { - "need_run": false - } - } - } - } - } -} diff --git a/examples/lr/test_hetero_lr_job_conf_advanced.json b/examples/lr/test_hetero_lr_job_conf_advanced.json deleted file mode 100644 index bae107126..000000000 --- a/examples/lr/test_hetero_lr_job_conf_advanced.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "dsl_version": "2", - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "guest": [ - 9999 - ], - "host": [ - 10000 - ], - "arbiter": [ - 10000 - ] - }, - "job_parameters": { - "common": { - "task_parallelism": 2, - "computing_partitions": 8, - "eggroll_run": { - "eggroll.session.processors.per.node": 2 - }, - "spark_run": { - "num-executors": 1, - "executor-cores": 2 - } - } - }, - "component_parameters": { - "common": { - "intersect_0": { - "intersect_method": "raw", - "sync_intersect_ids": true, - "only_output_key": false - }, - "hetero_lr_0": { - "penalty": "L2", - "optimizer": "rmsprop", - "alpha": 0.01, - "max_iter": 3, - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - } - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": true, - "label_name": "y", - "label_type": "int", - "output_format": "dense" - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": false, - "output_format": "dense" - } - } - } - } - } -} diff --git a/examples/lr/test_hetero_lr_job_dsl.json b/examples/lr/test_hetero_lr_job_dsl.json deleted file mode 100644 index 3e33c602c..000000000 --- a/examples/lr/test_hetero_lr_job_dsl.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "table" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.table" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "data_transform" - ] - }, - "need_deploy": true - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ] - } - }, - "hetero_feature_binning_0": { - "module": "HeteroFeatureBinning", - "input": { - "data": { - "data": [ - "intersection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "hetero_feature_binning" - ] - } - }, - "hetero_feature_selection_0": { - "module": "HeteroFeatureSelection", - "input": { - "data": { - "data": [ - "hetero_feature_binning_0.train" - ] - }, - "isometric_model": [ - "hetero_feature_binning_0.hetero_feature_binning" - ] - }, - "output": { - "data": [ - "train" - ], - "model": [ - "selected" - ] - } - }, - "hetero_lr_0": { - "module": "HeteroLR", - "input": { - "data": { - "train_data": [ - "hetero_feature_selection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "hetero_lr" - ] - } - }, - "evaluation_0": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "hetero_lr_0.train" - ] - } - }, - "output": { - "data": [ - "evaluate" - ] - } - } - } -} diff --git a/examples/model/bind_model_service.json b/examples/model/bind_model_service.json deleted file mode 100644 index 6423bcaca..000000000 --- a/examples/model/bind_model_service.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "service_id": "", - "initiator": { - "party_id": "10000", - "role": "guest" - }, - "role": { - "guest": [ - "10000" - ], - "host": [ - "10000" - ], - "arbiter": [ - "10000" - ] - }, - "job_parameters": { - "model_id": "arbiter-10000#guest-10000#host-10000#model", - "model_version": "2019081217340125761469" - }, - "servings": [ - ] -} diff --git a/examples/model/export.json b/examples/model/export.json new file mode 100644 index 000000000..5efb445af --- /dev/null +++ b/examples/model/export.json @@ -0,0 +1,7 @@ +{ + "model_id": "202307251326130289470", + "model_version": "0", + "role": "guest", + "party_id": "9999", + "path": "xxx/dir" +} \ No newline at end of file diff --git a/examples/model/export_model.json b/examples/model/export_model.json deleted file mode 100644 index 3969cac03..000000000 --- a/examples/model/export_model.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "role": "guest", - "party_id": 9999, - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "202006122009345696314", - "output_path": "/data/projects/fate/python/temp" -} diff --git a/examples/model/homo_convert_model.json b/examples/model/homo_convert_model.json deleted file mode 100644 index d239ed9e5..000000000 --- a/examples/model/homo_convert_model.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "role": "guest", - "party_id": 9999, - "model_id": "arbiter-9999#guest-9999#host-9998#model", - "model_version": "202110291115511918280" -} diff --git a/examples/model/homo_deploy_model.json b/examples/model/homo_deploy_model.json deleted file mode 100644 index aaf7959d2..000000000 --- a/examples/model/homo_deploy_model.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "service_id": "sklearn-lr", - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "202006122009345696314", - "role": "guest", - "party_id": 9999, - "component_name": "homo_lr_0", - "deployment_type": "kfserving", - "deployment_parameters": { - "protocol_version": "v1", - "namespace": "default", - "config_file_content": "", - "config_file": "", - "replace": false, - "skip_create_storage_secret": false, - "model_storage_type": "minio", - "model_storage_parameters": { - "endpoint": "minio:9000", - "access_key": "", - "secret_key": "", - "secure": false, - "region": "" - } - } -} \ No newline at end of file diff --git a/examples/model/import.json b/examples/model/import.json new file mode 100644 index 000000000..442302855 --- /dev/null +++ b/examples/model/import.json @@ -0,0 +1,5 @@ +{ + "model_id": "xxx", + "model_version": "xxx", + "path": "xxx" +} \ No newline at end of file diff --git a/examples/model/import_model.json b/examples/model/import_model.json deleted file mode 100644 index 74e091a88..000000000 --- a/examples/model/import_model.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "role": "guest", - "party_id": 9999, - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "202006122009345696314", - "file": "/data/projects/fate/python/temp/guest#9999#arbiter-10000#guest-9999#host-10000#model_202006122009345696314.zip", - "force_update": false -} diff --git a/examples/model/migrate_model.json b/examples/model/migrate_model.json deleted file mode 100644 index 81faa81c8..000000000 --- a/examples/model/migrate_model.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "job_parameters": { - "federated_mode": "SINGLE" - }, - "role": { - "guest": [ - 9999 - ], - "arbiter": [ - 10000 - ], - "host": [ - 10000 - ] - }, - "execute_party": { - "guest": [ - 9999 - ], - "arbiter": [ - 10000 - ], - "host": [ - 10000 - ] - }, - "migrate_initiator": { - "role": "guest", - "party_id": 99 - }, - "migrate_role": { - "guest": [ - 99 - ], - "arbiter": [ - 100 - ], - "host": [ - 100 - ] - }, - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "202110221748487862160" -} diff --git a/examples/model/publish_load_model.json b/examples/model/publish_load_model.json deleted file mode 100644 index e2ed5416e..000000000 --- a/examples/model/publish_load_model.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "initiator": { - "party_id": "10000", - "role": "guest" - }, - "role": { - "guest": [ - "10000" - ], - "host": [ - "10000" - ], - "arbiter": [ - "10000" - ] - }, - "job_parameters": { - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "202006122116502527621" - } -} diff --git a/examples/model/restore_model.json b/examples/model/restore_model.json deleted file mode 100644 index 0ef932002..000000000 --- a/examples/model/restore_model.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "role": "guest", - "party_id": 9999, - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "202006122009345696314" -} diff --git a/examples/model/store_model.json b/examples/model/store_model.json deleted file mode 100644 index baeb005cb..000000000 --- a/examples/model/store_model.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "role": "guest", - "party_id": 9999, - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "202006122009345696314", - "force_update": false -} diff --git a/examples/model/test_inference.py b/examples/model/test_inference.py deleted file mode 100644 index ea6a4b9ac..000000000 --- a/examples/model/test_inference.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*-coding:utf8 -*- -import requests -import argparse - -def inference(ids, ip, port): - url1 = f"http://{ip}:{port}/federation/1.0/inference" - for id in ids: - request_data_tmp = { - "head": { - "serviceId": "test_model_service", - "applyId": "209090900991", - }, - "body": { - "featureData": { - "phone_num": id, - }, - "sendToRemoteFeatureData": { - "device_type": "imei", - "phone_num": id, - "encrypt_type": "raw" - } - } - } - headers = {"Content-Type": "application/json"} - response = requests.post(url1, json=request_data_tmp, headers=headers) - print("url地址:", url1) - print("请求信息:\n", request_data_tmp) - print() - print("响应信息:\n", response.text) - print() - - -if __name__ == '__main__': - arg_parser = argparse.ArgumentParser() - arg_parser.add_argument("--ids", type=str, nargs='+', help="Data identification", required=False, - default=["123", "456"]) - arg_parser.add_argument("--ip", type=str, help="serving ip", required=False, default="127.0.0.1") - arg_parser.add_argument("--port", type=str, help="serving port", required=False, default="8059") - args = arg_parser.parse_args() - ids = args.ids - ip = args.ip - port = args.port - inference(ids=ids, ip=ip, port=port) \ No newline at end of file diff --git a/examples/other/register_provider.json b/examples/other/register_provider.json deleted file mode 100644 index 7f555a7f0..000000000 --- a/examples/other/register_provider.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "fate", - "version": "1.7.1", - "path": "./python/component_plugins/fateb/python/federatedml" -} \ No newline at end of file diff --git a/examples/other/update_parameters.json b/examples/other/update_parameters.json deleted file mode 100644 index 279869a2b..000000000 --- a/examples/other/update_parameters.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "job_parameters": { - }, - "component_parameters": { - "common": { - "hetero_lr_0": { - "alpha": 0.02, - "max_iter": 5 - } - } - } -} diff --git a/examples/permission/delete.json b/examples/permission/delete.json index 2f27fb8d7..2736324e3 100644 --- a/examples/permission/delete.json +++ b/examples/permission/delete.json @@ -1,10 +1,7 @@ { - "party_id": 10000, - "component": "reader,dataio", - "dataset": [ - { - "namespace": "experiment", - "name": "breast_hetero_guest" - } - ] + "party_id": "9999", + "component": "reader", + "dataset": [{ + "name": "xxx", "namespace": "xxx" + }] } \ No newline at end of file diff --git a/examples/permission/delete_all.json b/examples/permission/delete_all.json deleted file mode 100644 index b42a47da3..000000000 --- a/examples/permission/delete_all.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "party_id": 10000, - "component": "*", - "dataset": "*" -} \ No newline at end of file diff --git a/examples/permission/grant.json b/examples/permission/grant.json index 34cb583dc..2736324e3 100644 --- a/examples/permission/grant.json +++ b/examples/permission/grant.json @@ -1,14 +1,7 @@ { - "party_id": 10000, - "component": "reader,dataio", - "dataset": [ - { - "namespace": "experiment", - "name": "breast_hetero_guest" - }, - { - "namespace": "experiment", - "name": "breast_hetero_host" - } - ] + "party_id": "9999", + "component": "reader", + "dataset": [{ + "name": "xxx", "namespace": "xxx" + }] } \ No newline at end of file diff --git a/examples/permission/grant_all_component.json b/examples/permission/grant_all_component.json deleted file mode 100644 index 2ea5fd4e1..000000000 --- a/examples/permission/grant_all_component.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "party_id": 10000, - "component": "*" -} \ No newline at end of file diff --git a/examples/predict/test_predict_conf.json b/examples/predict/test_predict_conf.json deleted file mode 100644 index b0af9ac04..000000000 --- a/examples/predict/test_predict_conf.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "dsl_version": "2", - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "guest": [ - 9999 - ], - "host": [ - 10000 - ], - "arbiter": [ - 10000 - ] - }, - "job_parameters": { - "common": { - "job_type": "predict", - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "202109011220444359410" - } - }, - "component_parameters": { - "common": { - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": true, - "label_name": "y", - "label_type": "int", - "output_format": "dense" - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": false, - "output_format": "dense" - } - } - } - } - } -} diff --git a/examples/provider/provider_test_conf.json b/examples/provider/provider_test_conf.json deleted file mode 100644 index 9a952d38b..000000000 --- a/examples/provider/provider_test_conf.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "dsl_version": "2", - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "guest": [ - 9999 - ], - "host": [ - 10000 - ], - "arbiter": [ - 10000 - ] - }, - "job_parameters": { - "common": { - "task_parallelism": 2, - "computing_partitions": 8, - "task_cores": 4, - "auto_retries": 1 - } - }, - "component_parameters": { - "common": { - "intersection_0": { - "intersect_method": "raw", - "sync_intersect_ids": true, - "only_output_key": false - }, - "hetero_lr_0": { - "penalty": "L2", - "optimizer": "rmsprop", - "alpha": 0.01, - "max_iter": 3, - "batch_size": 320, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - } - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": true, - "label_name": "y", - "label_type": "int", - "output_format": "dense" - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": false, - "output_format": "dense" - }, - "evaluation_0": { - "need_run": false - } - } - } - } - }, - "provider": { - "common": { - "hetero_feature_binning_0": "fate@1.8.0" - }, - "role": { - "guest": { - "0": { - "data_transform_0": "fate@1.9.0" - } - }, - "host": { - "0": { - "data_transform_0": "fate@1.9.0" - } - } - } - } -} diff --git a/examples/provider/provider_test_dsl.json b/examples/provider/provider_test_dsl.json deleted file mode 100644 index 412b65539..000000000 --- a/examples/provider/provider_test_dsl.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "table" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.table" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "dataio" - ] - }, - "need_deploy": true - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ] - } - }, - "hetero_feature_binning_0": { - "module": "HeteroFeatureBinning", - "input": { - "data": { - "data": [ - "intersection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "hetero_feature_binning" - ] - } - }, - "hetero_feature_selection_0": { - "module": "HeteroFeatureSelection", - "input": { - "data": { - "data": [ - "hetero_feature_binning_0.train" - ] - }, - "isometric_model": [ - "hetero_feature_binning_0.hetero_feature_binning" - ] - }, - "output": { - "data": [ - "train" - ], - "model": [ - "selected" - ] - } - }, - "hetero_lr_0": { - "module": "HeteroLR", - "input": { - "data": { - "train_data": [ - "hetero_feature_selection_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "hetero_lr" - ] - } - }, - "evaluation_0": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "hetero_lr_0.train" - ] - } - }, - "output": { - "data": [ - "evaluate" - ] - } - } - } -} diff --git a/examples/provider/register.json b/examples/provider/register.json new file mode 100644 index 000000000..f9919beab --- /dev/null +++ b/examples/provider/register.json @@ -0,0 +1,9 @@ +{ + "name": "fate", + "device": "local", + "version": "2.0.0", + "metadata": { + "path": "xxx", + "venv": "xxx" + } +} \ No newline at end of file diff --git a/examples/simple/simple_dsl.json b/examples/simple/simple_dsl.json deleted file mode 100644 index b8e0dfb3b..000000000 --- a/examples/simple/simple_dsl.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "table" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.table" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "data_transform" - ] - }, - "need_deploy": true - }, - "hetero_feature_binning_0": { - "module": "HeteroFeatureBinning", - "input": { - "data": { - "data": [ - "data_transform_0.train" - ] - } - }, - "output": { - "data": [ - "train" - ], - "model": [ - "hetero_feature_binning" - ] - } - } - } -} diff --git a/examples/simple/simple_job_conf.json b/examples/simple/simple_job_conf.json deleted file mode 100644 index 44aef4eb7..000000000 --- a/examples/simple/simple_job_conf.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "dsl_version": "2", - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "guest": [ - 9999 - ], - "host": [ - 10000 - ], - "arbiter": [ - 10000 - ] - }, - "job_parameters": { - "common": { - "task_parallelism": 1, - "auto_retries": 1, - "computing_partitions": 8 - } - }, - "component_parameters": { - "common": { - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": true, - "label_name": "y", - "label_type": "int", - "output_format": "dense" - } - } - }, - "host": { - "0": { - "reader_0": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - }, - "data_transform_0": { - "with_label": false, - "output_format": "dense" - } - } - } - } - } -} diff --git a/examples/table_bind/bind_hdfs_table.json b/examples/table_bind/bind_hdfs_table.json deleted file mode 100644 index 39716b4f3..000000000 --- a/examples/table_bind/bind_hdfs_table.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "namespace": "experiment", - "name": "breast_hetero_guest", - "engine": "HDFS", - "address": { - "name_node": "hdfs://fate-cluster", - "path": "/data/breast_hetero_guest.csv" - }, - "id_delimiter": ",", - "head": 1, - "partitions": 10 -} diff --git a/examples/table_bind/bind_mysql_table.json b/examples/table_bind/bind_mysql_table.json deleted file mode 100644 index 84b4deb0d..000000000 --- a/examples/table_bind/bind_mysql_table.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "engine": "MYSQL", - "address": { - "user": "fate", - "passwd": "fate", - "host": "127.0.0.1", - "port": 3306, - "db": "experiment", - "name": "breast_hetero_guest" - }, - "namespace": "experiment", - "name": "breast_hetero_guest", - "head": 1, - "id_delimiter": ",", - "partitions": 10, - "id_column": "id", - "feature_column": "y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9" -} \ No newline at end of file diff --git a/examples/table_bind/bind_mysql_table_by_connector_name.json b/examples/table_bind/bind_mysql_table_by_connector_name.json deleted file mode 100644 index f04c5a291..000000000 --- a/examples/table_bind/bind_mysql_table_by_connector_name.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "engine": "MYSQL", - "address": { - "connector_name": "test", - "db": "experiment", - "name": "breast_hetero_guest" - }, - "namespace": "experiment", - "name": "breast_hetero_guest", - "head": 1, - "id_delimiter": ",", - "partitions": 10, - "id_column": "id", - "feature_column": "y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9" -} \ No newline at end of file diff --git a/examples/table_bind/bind_mysql_table_with_anonymous.json b/examples/table_bind/bind_mysql_table_with_anonymous.json deleted file mode 100644 index 503481fb3..000000000 --- a/examples/table_bind/bind_mysql_table_with_anonymous.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "engine": "MYSQL", - "address": { - "user": "fate", - "passwd": "fate", - "host": "127.0.0.1", - "port": 3306, - "db": "experiment", - "name": "breast_hetero_guest" - }, - "namespace": "experiment", - "name": "breast_hetero_guest", - "head": 1, - "id_delimiter": ",", - "partitions": 10, - "id_column": "id", - "feature_column": "y,x0,x1,x2,x3,x4,x5,x6,x7,x8,x9", - "with_meta": true, - "meta": { - "input_format": "dense", - "tag_with_value": false, - "tag_value_delimiter": ":", - "with_match_id": false, - "id_list": null, - "id_range": 0, - "exclusive_data_type": null, - "data_type": "float64", - "with_label": false, - "label_name": "y", - "label_type": "int" - } -} \ No newline at end of file diff --git a/examples/table_bind/bind_path_table.json b/examples/table_bind/bind_path_table.json deleted file mode 100644 index f9db8a3a9..000000000 --- a/examples/table_bind/bind_path_table.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "namespace": "xxx", - "name": "xxx", - "engine": "PATH", - "address": { - "path": "xxx" - } -} diff --git a/examples/toy/toy_example_conf.json b/examples/toy/toy_example_conf.json deleted file mode 100644 index 5a95d948d..000000000 --- a/examples/toy/toy_example_conf.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "dsl_version": 2, - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "job_parameters": { - "common": { - }, - "role": { - "guest": { - "0": { - "task_cores": 2 - } - }, - "host": { - "0": { - "task_cores": 3 - } - } - } - }, - "role": { - "guest": [ - 9999 - ], - "host": [ - 10000 - ] - }, - "component_parameters": { - "role": { - "guest": { - "0": { - "secure_add_example_0": { - "seed": 123 - } - } - }, - "host": { - "secure_add_example_0": { - "seed": 321 - } - } - }, - "common": { - "secure_add_example_0": { - "partition": 48, - "data_num": 1000 - } - } - } -} diff --git a/examples/toy/toy_example_dsl.json b/examples/toy/toy_example_dsl.json deleted file mode 100644 index fdbeb8b5f..000000000 --- a/examples/toy/toy_example_dsl.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "components": { - "secure_add_example_0": { - "module": "SecureAddExample" - } - } -} diff --git a/examples/transformer/transformer_guest.json b/examples/transformer/transformer_guest.json new file mode 100644 index 000000000..a7623f271 --- /dev/null +++ b/examples/transformer/transformer_guest.json @@ -0,0 +1,8 @@ +{ + "namespace": "experiment", + "name": "breast_homo_guest", + "data_warehouse": { + "name": "7e1441e4-9413-11ee-9bb0-16b977118319", + "namespace": "upload" + } +} diff --git a/examples/transformer/transformer_host.json b/examples/transformer/transformer_host.json new file mode 100644 index 000000000..d9bed8fab --- /dev/null +++ b/examples/transformer/transformer_host.json @@ -0,0 +1,8 @@ +{ + "namespace": "experiment", + "name": "breast_homo_host", + "data_warehouse": { + "name": "9fa4905c-9413-11ee-9bb0-16b977118319", + "namespace": "upload" + } +} diff --git a/examples/upload/upload_guest.json b/examples/upload/upload_guest.json index c0c3312c9..b7526c89a 100644 --- a/examples/upload/upload_guest.json +++ b/examples/upload/upload_guest.json @@ -1,8 +1,13 @@ { "file": "examples/data/breast_hetero_guest.csv", - "id_delimiter": ",", - "head": 1, - "partition": 4, + "head": true, + "partitions": 16, + "extend_sid": true, + "meta": { + "delimiter": ",", + "label_name": "y", + "match_id_name": "id" + }, "namespace": "experiment", - "table_name": "breast_hetero_guest" + "name": "breast_hetero_guest" } diff --git a/examples/upload/upload_guest_with_meta.json b/examples/upload/upload_guest_with_meta.json deleted file mode 100644 index b33eaa601..000000000 --- a/examples/upload/upload_guest_with_meta.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "file": "examples/data/breast_hetero_guest.csv", - "id_delimiter": ",", - "head": 1, - "partition": 4, - "namespace": "experiment", - "table_name": "breast_hetero_anonymous_guest", - "with_meta": true, - "meta": { - "with_label": true, - "label_name": "y" - } -} diff --git a/examples/upload/upload_homo_guest.json b/examples/upload/upload_homo_guest.json deleted file mode 100644 index 2df54b4e0..000000000 --- a/examples/upload/upload_homo_guest.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "file": "examples/data/breast_homo_guest.csv", - "head": 1, - "partition": 10, - "namespace": "experiment", - "table_name": "breast_homo_guest" -} diff --git a/examples/upload/upload_homo_host.json b/examples/upload/upload_homo_host.json deleted file mode 100644 index 83f12e76c..000000000 --- a/examples/upload/upload_homo_host.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "file": "examples/data/breast_homo_host.csv", - "head": 1, - "partition": 10, - "namespace": "experiment", - "table_name": "breast_homo_host" -} diff --git a/examples/upload/upload_homo_test.json b/examples/upload/upload_homo_test.json deleted file mode 100644 index 0d4d3fcfd..000000000 --- a/examples/upload/upload_homo_test.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "file": "examples/data/breast_homo_test.csv", - "head": 1, - "partition": 10, - "namespace": "experiment", - "table_name": "breast_homo_test" -} diff --git a/examples/upload/upload_host.json b/examples/upload/upload_host.json index 7246874c5..efb6fec36 100644 --- a/examples/upload/upload_host.json +++ b/examples/upload/upload_host.json @@ -1,8 +1,12 @@ { "file": "examples/data/breast_hetero_host.csv", - "id_delimiter": ",", - "head": 1, - "partition": 10, + "head": true, + "partitions": 16, + "extend_sid": true, + "meta": { + "delimiter": ",", + "match_id_name": "id" + }, "namespace": "experiment", - "table_name": "breast_hetero_host" + "name": "breast_hetero_host" } diff --git a/examples/upload/upload_host_with_meta.json b/examples/upload/upload_host_with_meta.json deleted file mode 100644 index 647728a1a..000000000 --- a/examples/upload/upload_host_with_meta.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "file": "examples/data/breast_hetero_host.csv", - "id_delimiter": ",", - "head": 1, - "partition": 4, - "namespace": "experiment", - "table_name": "breast_hetero_anonymous_host", - "with_meta": true, - "meta": { - "with_label": false - } -} diff --git a/examples/upload/upload_to_eggroll.json b/examples/upload/upload_to_eggroll.json deleted file mode 100644 index c0c3312c9..000000000 --- a/examples/upload/upload_to_eggroll.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "file": "examples/data/breast_hetero_guest.csv", - "id_delimiter": ",", - "head": 1, - "partition": 4, - "namespace": "experiment", - "table_name": "breast_hetero_guest" -} diff --git a/examples/upload/upload_to_hdfs.json b/examples/upload/upload_to_hdfs.json deleted file mode 100644 index 9e87b7d0c..000000000 --- a/examples/upload/upload_to_hdfs.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "file": "examples/data/breast_hetero_guest.csv", - "id_delimiter": ",", - "head": 1, - "partition": 10, - "namespace": "experiment", - "table_name": "breast_hetero_guest", - "storage_engine": "HDFS" -} \ No newline at end of file diff --git a/examples/upload/upload_to_hive.json b/examples/upload/upload_to_hive.json deleted file mode 100644 index 5e0c85295..000000000 --- a/examples/upload/upload_to_hive.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "file": "examples/data/breast_hetero_host.csv", - "id_delimiter": ",", - "head": 1, - "partition": 10, - "namespace": "experiment", - "table_name": "breast_hetero_guest", - "storage_engine": "HIVE", - "storage_address": { - "host": "127.0.0.1", - "port": 10000, - "user": "fate", - "passwd": "fate", - "database": "experiment", - "name": "breast_hetero_guest", - "auth_mechanism": "PLAIN" - } -} diff --git a/examples/upload/upload_to_localfs.json b/examples/upload/upload_to_localfs.json deleted file mode 100644 index ba35bcc7f..000000000 --- a/examples/upload/upload_to_localfs.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "file": "examples/data/breast_hetero_guest.csv", - "id_delimiter": ",", - "head": 1, - "partition": 4, - "namespace": "experiment", - "table_name": "breast_hetero_guest", - "task_cores": 2, - "storage_engine": "LOCALFS" -} diff --git a/examples/upload/upload_with_meta.json b/examples/upload/upload_with_meta.json deleted file mode 100644 index f970c21f3..000000000 --- a/examples/upload/upload_with_meta.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "file": "xxx", - "id_delimiter": ",", - "head": 1, - "partition": 4, - "namespace": "experiment", - "table_name": "xxx", - "with_meta": true, - "meta": { - "input_format": "dense", - "tag_with_value": false, - "tag_value_delimiter": ":", - "with_match_id": false, - "id_list": null, - "id_range": 0, - "exclusive_data_type": null, - "data_type": "float64", - "with_label": false, - "label_name": "y", - "label_type": "int" - } -} diff --git a/examples/writer/external_storage.json b/examples/writer/external_storage.json deleted file mode 100644 index 42335a77d..000000000 --- a/examples/writer/external_storage.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "table_name": "name1", - "namespace": "namespace1", - "storage_engine": "MYSQL", - "address": { - "user": "fate", - "passwd": "fate_dev", - "host": "127.0.0.1", - "port": 3306, - "db": "namespace2", - "name": "name2" - } -} \ No newline at end of file diff --git a/examples/writer/save_to_new_table.json b/examples/writer/save_to_new_table.json deleted file mode 100644 index 0385378bb..000000000 --- a/examples/writer/save_to_new_table.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "table_name": "name1", - "namespace": "namespace1", - "output_name": "name2", - "output_namespace": "namespace2" -} \ No newline at end of file diff --git a/fateflow.env b/fateflow.env new file mode 100644 index 000000000..3d358ab8a --- /dev/null +++ b/fateflow.env @@ -0,0 +1,3 @@ +FATE=2.0.0 +FATE_FLOW=2.0.0 +PYTHON=3.8 \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 18d5dd94a..8b3f7a461 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,25 +9,9 @@ nav: - Home: index.md - Docs: #- ... | flat | *.md - - document_navigation.md - fate_flow.md - - fate_flow_data_access.md - - fate_flow_component_registry.md - - fate_flow_job_scheduling.md - - fate_flow_resource_management.md - - fate_flow_tracking.md - - fate_flow_monitoring.md - - fate_flow_model_registry.md - - fate_flow_authority_management.md - - fate_flow_permission_management.md - - fate_flow_server_operation.md - - fate_flow_service_registry.md - - fate_flow_model_migration.md - - fate_flow_client.md - - fate_flow_http_api_call_demo.md - - configuration_instruction.md - - system_operational.md - - faq.md + - quick_start.md + - system_conf.md - API: swagger/index.md theme: @@ -64,7 +48,13 @@ plugins: - i18n: default_language: en languages: - zh: 中文 + - locale: zh + name: 中文 + build: true + - locale: en + name: English + build: true + default: true - markdown-include-snippet: base_path: doc diff --git a/proto/generate_proto_buffer.sh b/proto/generate_proto_buffer.sh new file mode 100644 index 000000000..ca0cf34fd --- /dev/null +++ b/proto/generate_proto_buffer.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Constants +BASEDIR=$(dirname "$0") +PROTO_DIR="proto" +TARGET_DIR="../python/ofx/api/proto" + +# Function to display help +show_help() { + echo "Usage: $(basename "$0") [options]" + echo "Options:" + echo " -h, --help Show this help message" + echo " -r, --recursive Process directories recursively" + echo " -c, --clean Clean target directory before generation" +} + +# Function to check if grpc_tools is installed +check_grpc_tools_installed() { + python3 -c "import grpc_tools.protoc" 2>/dev/null + if [ $? -ne 0 ]; then + echo "grpc_tools is not installed. Please install it using 'python3 -m pip install grpcio-tools'" + exit 1 + fi +} + +# Function to generate stubs +generate() { + if grep -q "^service " "$3"; then + # Generate both Python and gRPC stubs + python3 -m grpc_tools.protoc -I"$1" \ + --python_out="$2" \ + --grpc_python_out="$2" \ + --mypy_out="$2" \ + "$3" + else + # Generate only Python stubs + python3 -m grpc_tools.protoc -I"$1" \ + --python_out="$2" \ + --mypy_out="$2" \ + "$3" + fi +} + +# Function to clean target directory +clean_target() { + rm -rf "$TARGET_DIR"/* +} + +# Function to generate stubs for all .proto files +generate_all() { + local recursive_flag=$1 + for sub_dir in "$BASEDIR"/*; do + if [ -d "$sub_dir" ] && [ "$sub_dir" != "$BASEDIR/$PROTO_DIR" ]; then + for proto in "$sub_dir"/*.proto; do + generate "$sub_dir" "$TARGET_DIR/${sub_dir##*/}" "$proto" + done + # Handle recursive flag + [ "$recursive_flag" == "true" ] && find "$sub_dir" -type d -exec bash -c 'generate_all "$1" "$2"' _ {} \; + fi + done +} + +# Parse command line options +recursive=false +clean=false +while [ $# -gt 0 ]; do + case $1 in + -h|--help) show_help; exit 0 ;; + -r|--recursive) recursive=true ;; + -c|--clean) clean=true ;; + *) echo "Unknown option: $1"; show_help; exit 1 ;; + esac + shift +done + +# Main execution +check_grpc_tools_installed +cd "$BASEDIR" || { echo "Failed to change directory to $BASEDIR"; exit 1; } +[ "$clean" == "true" ] && clean_target +generate_all "$recursive" +echo "Generation completed" diff --git a/proto/rollsite/basic-meta.proto b/proto/rollsite/basic-meta.proto new file mode 100644 index 000000000..8be575dcd --- /dev/null +++ b/proto/rollsite/basic-meta.proto @@ -0,0 +1,88 @@ +/* + * Copyright 2019 The Eggroll Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package com.webank.ai.eggroll.api.core; + +// network endpoint +message Endpoint { + string ip = 1; // ip address + int32 port = 2; // port + string hostname = 3; // host name +} + +message Endpoints { + repeated Endpoint endpoints = 1; +} + +// general container for serialized data +message Data { + bool isNull = 1; // whether the data is actually 'null' (e.g. null in Java, None in python, NULL in c/c++ + string hostLanguage = 2; // the host language which serializes it + string type = 3; // data type in host language + bytes data = 4; // actual data serialized in bytes +} + +// general container for data list +message RepeatedData { + repeated Data datalist = 1; // list of data +} + +// general message for a call request +message CallRequest { + bool isAsync = 1; // whether the call is async. ignored in phase 1 + int64 timeout = 2; // in ms + string command = 3; // call command. ignored in phase 1 + Data param = 4; // call input param +} + +// general message for a call response +// todo: merge call response with Return Status +message CallResponse { + ReturnStatus returnStatus = 1; // return status + Data result = 2; // call result +} + +message Job { + string jobId = 1; + string name = 2; +} + +message Task { + Job job = 1; + int64 taskId = 2; + int64 tableId = 3; +} + +// reserved for driver async call result +message Result { + Task task = 1; + int64 resultId = 2; +} + +// generic return status +message ReturnStatus { + int32 code = 1; + string message = 2; +} + +message SessionInfo { + string sessionId = 1; + map computingEngineConf = 2; + string namingPolicy = 3; + string tag = 4; +} \ No newline at end of file diff --git a/proto/rollsite/proxy.proto b/proto/rollsite/proxy.proto new file mode 100644 index 000000000..b3932505e --- /dev/null +++ b/proto/rollsite/proxy.proto @@ -0,0 +1,114 @@ +/* + * Copyright 2019 The Eggroll Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +import "basic-meta.proto"; + +package com.webank.ai.eggroll.api.networking.proxy; + +// metadata of event +message Model { + string name = 1; + string dataKey = 2; +} + +// metadata of task +message Task { + string taskId = 1; + Model model = 2; +} + +message Topic { + string name = 1; + string partyId = 2; + string role = 3; + com.webank.ai.eggroll.api.core.Endpoint callback = 4; // implication of pub/sub model, necessary for http-based senario +} + +// task admin command +message Command { + string name = 1; +} + +message Conf { + int64 overallTimeout = 1; // total timeout, in ms + int64 completionWaitTimeout = 2; // timeout for waiting for complete, in ms + int64 packetIntervalTimeout = 3; // timeout for packet interval, in ms + int32 maxRetries = 4; +} + +// metadata used for network data transfer +message Metadata { + Task task = 1; // task description + Topic src = 2; // source topic + Topic dst = 3; // destincation topic + Command command = 4; // task managing command (if any) + string operator = 5; // model operator + int64 seq = 6; // stream seq (reserved) + int64 ack = 7; // stream ack (reserved) + Conf conf = 8; // operation config + bytes ext = 9; + string version = 100; +} + +// includes key and value field, supporting sequential and random data transfer +message Data { + string key = 1; // compatible with list / dict + bytes value = 2; // actual value +} + +// data streaming packet +message Packet { + Metadata header = 1; // packet header + Data body = 2; // packet body +} + +// returned by service heartbeat to decide next operation +enum Operation { + START = 0; + RUN = 1; + STOP = 2; + KILL = 3; + GET_DATA = 4; + PUT_DATA = 5; +} + +// response of heartbeat +message HeartbeatResponse { + Metadata header = 1; + Operation operation = 2; +} + +message PollingFrame { + string method = 1; + int64 seq = 2; + Metadata metadata = 10; + Packet packet = 20; + string desc = 30; +} + +// data transfer service +service DataTransferService { + rpc push (stream Packet) returns (Metadata); + rpc pull (Metadata) returns (stream Packet); + rpc unaryCall (Packet) returns (Packet); + rpc polling (stream PollingFrame) returns (stream PollingFrame); +} + +service RouteService { + rpc query (Topic) returns (com.webank.ai.eggroll.api.core.Endpoint); +} diff --git a/python/fate_flow/__init__.py b/python/fate_flow/__init__.py index 9c1f76acf..8d9ccccee 100644 --- a/python/fate_flow/__init__.py +++ b/python/fate_flow/__init__.py @@ -12,24 +12,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -import os - -from fate_flow.entity.types import ProcessRole - - -def set_env(): - """ - project_base = os.path.abspath( - os.path.join( - os.path.dirname(os.path.realpath(__file__)), - os.pardir, - os.pardir, - ) - ) - os.environ.setdefault("FATE_DEPLOY_BASE", project_base) - """ - os.environ.setdefault("PROCESS_ROLE", ProcessRole.DRIVER.value) - - -set_env() +from ._info import __provider__, __version__ diff --git a/python/fate_flow/_info.py b/python/fate_flow/_info.py new file mode 100644 index 000000000..319c98fb7 --- /dev/null +++ b/python/fate_flow/_info.py @@ -0,0 +1,16 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +__version__ = "2.0.0" +__provider__ = "fate_flow" diff --git a/python/fate_flow/adapter/__init__.py b/python/fate_flow/adapter/__init__.py new file mode 100644 index 000000000..35df150dc --- /dev/null +++ b/python/fate_flow/adapter/__init__.py @@ -0,0 +1,2 @@ +from ._loader import init_adapter, load_adapter_apps +from ._controller import AdapterJobController diff --git a/python/fate_flow/adapter/_controller.py b/python/fate_flow/adapter/_controller.py new file mode 100644 index 000000000..0b1794743 --- /dev/null +++ b/python/fate_flow/adapter/_controller.py @@ -0,0 +1,36 @@ +from fate_flow.adapter._loader import load_bridge_module +from fate_flow.entity.spec.flow import SubmitJobInput, SubmitJobOutput, QueryJobInput, QueryJobOutput, StopJobInput, \ + StopJobOutput, QueryTaskOutput, QueryTaskInput + + +class AdapterJobController(object): + def __init__(self, protocol_name): + packages = load_bridge_module(protocol_name=protocol_name) + self.controller_adapter = getattr(packages, "JobController") + + def create_job(self, submit_job_input: SubmitJobInput) -> SubmitJobOutput: + return self.controller_adapter.create_job(submit_job_input) + + def query_job(self, query_job_input: QueryJobInput) -> QueryJobOutput: + return self.controller_adapter.query_job(query_job_input) + + def stop_job(self, stop_job_input: StopJobInput) -> StopJobOutput: + return self.controller_adapter.stop_job(stop_job_input) + + def query_task(self, query_task_input: QueryTaskInput) -> QueryTaskOutput: + return self.controller_adapter.query_task(query_task_input) + + def rerun_job(self): + pass + + @staticmethod + def query_output_data(): + return {} + + @staticmethod + def query_output_model(): + return {} + + @staticmethod + def query_output_metric(): + return {} diff --git a/python/fate_flow/adapter/_loader.py b/python/fate_flow/adapter/_loader.py new file mode 100644 index 000000000..399ecc686 --- /dev/null +++ b/python/fate_flow/adapter/_loader.py @@ -0,0 +1,83 @@ +import inspect +import os +from importlib import import_module +from pathlib import Path + +from fate_flow.scheduler import SchedulerABC +from fate_flow.utils.cron import Cron + + +def load_adapter_name(): + return [ + name for name in os.listdir(Path(__file__).parent) + if os.path.isdir(os.path.join(Path(__file__).parent, name)) and not name.endswith("__") + ] + + +adapter_list = load_adapter_name() + + +def get_app_module(page_path): + page_name = page_path.stem.rstrip('app').rstrip("_") + return ".".join([get_module(page_path), page_name]) + + +def get_module(page_path): + module_name = '.'.join(page_path.parts[page_path.parts.index('fate_flow') + 2:-1]) + return module_name + + +def load_bridge_module(module_name="bridge", protocol_name=None): + for name in adapter_list: + if protocol_name == name: + path = Path(__file__).parent / name / module_name + adapter_module = get_module(path) + return import_module(".".join([adapter_module, module_name])) + + +def load_adapter_packages(module_name): + packages = [] + for name in adapter_list: + path = Path(__file__).parent / name / module_name + adapter_module = get_module(path) + try: + packages.append(import_module(".".join([adapter_module, module_name]))) + except Exception as e: + print(e) + pass + return packages + + +def load_scheduler(): + packages = load_adapter_packages("scheduler") + for package in packages: + members = inspect.getmembers(package) + class_list = [member[1] for member in members if inspect.isclass(member[1])] + for _class in class_list: + if issubclass(_class, SchedulerABC): + # start scheduler + _class(interval=2 * 1000).start() + + elif issubclass(_class, Cron): + print(_class) + _class(interval=5 * 1000).start() + + +def load_db(): + load_adapter_packages("db") + + +def load_adapter_apps(register_page, search_pages_path): + urls_dict = {} + for name in adapter_list: + path = Path(__file__).parent / name / "apps" + version = getattr(import_module(get_app_module(path)), "__version__", None) + before_request_func = getattr(import_module(get_app_module(path)), "before_request", None) + urls_dict[name] = [register_page(path, func=before_request_func, prefix=version) for path in + search_pages_path(path)] + return urls_dict + + +def init_adapter(): + load_db() + load_scheduler() diff --git a/python/fate_flow/adapter/bfia/__init__.py b/python/fate_flow/adapter/bfia/__init__.py new file mode 100644 index 000000000..68598ea67 --- /dev/null +++ b/python/fate_flow/adapter/bfia/__init__.py @@ -0,0 +1,5 @@ +__all__ = ["apps"] + +from fate_flow.adapter.bfia.runtime_config import BfiaRuntimeConfig + +BfiaRuntimeConfig.init() diff --git a/python/fate_flow/adapter/bfia/apps/__init__.py b/python/fate_flow/adapter/bfia/apps/__init__.py new file mode 100644 index 000000000..5cb457445 --- /dev/null +++ b/python/fate_flow/adapter/bfia/apps/__init__.py @@ -0,0 +1,6 @@ +__version__ = "v1" + + +def before_request(): + # 签名、认证 + pass diff --git a/python/fate_flow/adapter/bfia/apps/interconn_app.py b/python/fate_flow/adapter/bfia/apps/interconn_app.py new file mode 100644 index 000000000..0e32fd1ba --- /dev/null +++ b/python/fate_flow/adapter/bfia/apps/interconn_app.py @@ -0,0 +1,143 @@ +from webargs import fields + +from fate_flow.adapter.bfia import apps +from fate_flow.adapter.bfia.scheduler import BfiaScheduler +from fate_flow.adapter.bfia.utils.api_utils import BfiaAPI as API +from fate_flow.adapter.bfia.utils.entity.code import ReturnCode +from fate_flow.adapter.bfia.wheels.job import BfiaJobController +from fate_flow.adapter.bfia.wheels.task import BfiaTaskController + +page_name = 'interconn' + + +# scheduler +# 发起方向调度方发送创建作业请求时调用的接口 +@manager.route('/schedule/job/create_all', methods=['POST']) +@API.Input.json(flow_id=fields.String(required=False)) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(dag=fields.Dict(required=True)) +@API.Input.json(config=fields.Dict(required=True)) +@API.Input.json(old_job_id=fields.String(required=False)) +def create_job_all(job_id, dag, config, flow_id=None, old_job_id=None): + dag_schema = dict( + dag=dict( + dag=dag, + config=config, + flow_id=flow_id, + old_job_id=old_job_id + ), + schema_version=apps.__version__ + ) + submit_result = BfiaScheduler.create_all_job(job_id=job_id, dag=dag_schema) + return API.Output.json(**submit_result) + + +# scheduler +# 发起方向调度方发送停止作业请求时调用的接口 +@manager.route('/schedule/job/stop_all', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(task_name=fields.String(required=False)) +def stop_job_all(job_id, task_name=None): + BfiaScheduler.stop_all_job(job_id=job_id, task_name=task_name) + return API.Output.json() + + +# scheduler +# 某个参与方向调度方发起查询作业状态时调用的接口 +@manager.route('/schedule/job/status_all', methods=['GET']) +@API.Input.json(job_id=fields.String(required=True)) +def get_job_status_all(job_id): + data = BfiaScheduler.query_job_status(job_id=job_id) + return API.Output.json(data=data) + + +# scheduler +# 某个参与方向调度方或发起方作业审批结果时调用的接口 +@manager.route('/schedule/job/audit_confirm', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(status=fields.String(required=True)) +def audit_confirm(job_id, status): + status = BfiaScheduler.audit_confirm(job_id, status) + if status: + return API.Output.json() + else: + return API.Output.json(code=ReturnCode.FAILED) + + +# scheduler +# 任意参与方向发起方或调度方向推送任务回调信息时调用的接口 +@manager.route('/schedule/task/callback', methods=['POST']) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(status=fields.String(required=True)) +@API.Input.headers(x_node_id=fields.String(required=True)) +def callback_task(task_id, role, status, x_node_id): + status = BfiaScheduler.callback_task(task_id, role, status, x_node_id) + if status: + return API.Output.json() + return API.Output.json(code=ReturnCode.FAILED) + + +# partner +# 发起方或调度方向所有参与方发送创建作业请求时调用的接口 +@manager.route('/schedule/job/create', methods=['POST']) +@API.Input.json(flow_id=fields.String(required=False)) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(dag=fields.Dict(required=True)) +@API.Input.json(config=fields.Dict(required=True)) +@API.Input.json(old_job_id=fields.String(required=False)) +def create_job(job_id, dag, config, flow_id="", old_job_id=""): + dag_schema = dict( + dag=dict( + dag=dag, + config=config, + flow_id=flow_id, + old_job_id=old_job_id + ), + schema_version=apps.__version__ + ) + BfiaJobController.create_local_jobs(job_id=job_id, dag=dag_schema) + return API.Output.json() + + +# partner +# 发起方或调度方向所有参与方发送启动作业请求时调用的接口，每个参与方只会被请求一次，当某个参与方包含多个角色时， +# 由参与方自己根据作业配置将不同角色的任务启动 +@manager.route('/schedule/job/start', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +def start_job(job_id): + BfiaJobController.start_job(job_id=job_id) + return API.Output.json() + + +# partner +# 发起方或调度方向所有参与方发送启动任务请求时调用的接口 +@manager.route('/schedule/task/start', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_name=fields.String(required=True)) +def start_task(job_id, task_id, task_name): + status = BfiaTaskController.start_tasks(job_id, task_id, task_name) + if status: + return API.Output.json() + return API.Output.json(code=ReturnCode.FAILED) + + +# partner +# 发起方或调度方向所有参与方发送停止作业请求时调用的接口 +@manager.route('/schedule/job/stop', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(task_name=fields.String(required=False)) +def stop_job(job_id, task_name=None): + BfiaJobController.stop_local_jobs(job_id, task_name) + return API.Output.json() + + +# partner +# 发起方或调度方向所有参与方查询任务回调信息时调用的接口 +@manager.route('/schedule/task/poll', methods=['POST']) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +def poll_task(task_id, role): + status = BfiaTaskController.poll_task(task_id, role) + return API.Output.json(data={"status": status}) diff --git a/python/fate_flow/adapter/bfia/apps/platform_app.py b/python/fate_flow/adapter/bfia/apps/platform_app.py new file mode 100644 index 000000000..c205eaa4e --- /dev/null +++ b/python/fate_flow/adapter/bfia/apps/platform_app.py @@ -0,0 +1,80 @@ +from webargs import fields + +from fate_flow.adapter.bfia.wheels.job import BfiaJobController +from fate_flow.adapter.bfia.utils.api_utils import BfiaAPI as API +from fate_flow.adapter.bfia.wheels.task import BfiaTaskController + +page_name = 'platform' + + +# 发起方发起创建作业配置，并发送给己方调度层时调用的接口 +@manager.route('/schedule/job/create_all', methods=['POST']) +@API.Input.json(flow_id=fields.String(required=False)) +@API.Input.json(dag=fields.Dict(required=True)) +@API.Input.json(config=fields.Dict(required=True)) +@API.Input.json(old_job_id=fields.String(required=False)) +def create_job(dag, config, flow_id="", old_job_id=""): + job_id = BfiaJobController.request_create_job(dag, config, flow_id, old_job_id) + return API.Output.json(data={"job_id": job_id}) + + +# 发起方向己方调度层发起停止作业时调用的接口 +@manager.route('/schedule/job/stop_all', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +def stop_job(job_id): + BfiaJobController.request_stop_job(job_id) + return API.Output.json() + + +# 发起方向己方调度层发起停止任务时调用的接口 +@manager.route('/schedule/job/stop_task', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(task_name=fields.String(required=True)) +def stop_task(job_id, task_name): + BfiaTaskController.stop_local_task(job_id, task_name) + return API.Output.json() + + +# 某个参与方向己方调度层发起查询作业列表时调用的接口 +@manager.route('/schedule/job/query_job_list', methods=['GET']) +@API.Input.json(flow_id=fields.String(required=True)) +def query_job_list(flow_id): + job_list = BfiaJobController.query_job_status(flow_id=flow_id) + return API.Output.json(job_list=job_list) + + +# 某个参与方向调度方发起查询作业状态时调用的接口 +@manager.route('/schedule/job/status_all', methods=['GET']) +@API.Input.json(job_id=fields.String(required=True)) +def get_job_status(job_id): + status = BfiaTaskController.query_tasks_status(job_id) + return API.Output.json(status=status) + + +# 某个参与方节点上层服务向调度层获取任务运行日志行数时调用的接口 +@manager.route('/schedule/task/get_log_line', methods=['GET']) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(log_level=fields.String(required=True)) +def get_log_line(task_id, log_level): + return API.Output.json(num=0) + + +# 某个参与方节点上层服务向调度层获取任务运行日志内容时调用的接口 +@manager.route('/schedule/task/get_log', methods=['GET']) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(log_level=fields.String(required=True)) +@API.Input.json(start=fields.Int(required=True)) +@API.Input.json(length=fields.Int(required=False)) +def get_log(task_id, log_level, start, length=None): + return API.Output.json(data=[]) + + +# 某个参与方的算法组件层将任务运行的状态等回调信息推送给调度层时调用的接口 +@manager.route('/schedule/task/callback', methods=['POST']) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(status=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +def task_callback(task_id, status, role): + role = role.split(".")[0] + BfiaTaskController.callback_task(task_id, status, role) + return API.Output.json() diff --git a/python/fate_flow/adapter/bfia/bridge/__init__.py b/python/fate_flow/adapter/bfia/bridge/__init__.py new file mode 100644 index 000000000..68bb34e1d --- /dev/null +++ b/python/fate_flow/adapter/bfia/bridge/__init__.py @@ -0,0 +1,3 @@ +from .job import JobController + +__all__ = ["JobController"] diff --git a/python/fate_flow/adapter/bfia/bridge/job.py b/python/fate_flow/adapter/bfia/bridge/job.py new file mode 100644 index 000000000..68019ce0e --- /dev/null +++ b/python/fate_flow/adapter/bfia/bridge/job.py @@ -0,0 +1,68 @@ +from fate_flow.adapter.bfia.translator.component_spec import BFIAComponentSpec +from fate_flow.adapter.bfia.translator.dsl_translator import Translator +from fate_flow.adapter.bfia.utils.entity.status import StatusSet as BfiaJobStatus +from fate_flow.adapter.bfia.wheels.job import BfiaJobController +from fate_flow.entity.spec.flow import SubmitJobInput, QueryJobInput, QueryJobOutput, StopJobInput, StopJobOutput, \ + QueryTaskOutput, QueryTaskInput, SubmitJobOutput +from fate_flow.entity.types import JobStatus +from fate_flow.manager.service.provider_manager import ProviderManager + + +class JobController(object): + @staticmethod + def create_job(submit_job_input: SubmitJobInput): + dag_schema = submit_job_input.dag_schema + components_desc = {} + for name, desc in ProviderManager.query_component_description(protocol=dag_schema.kind).items(): + components_desc[name] = BFIAComponentSpec.parse_obj(desc) + bfia_dag = Translator.translate_dag_to_bfia_dag(dag_schema, components_desc) + job_id = BfiaJobController.request_create_job( + bfia_dag.dag.dag.dict(exclude_defaults=True), + bfia_dag.dag.config.dict(exclude_defaults=True), + bfia_dag.dag.flow_id, + bfia_dag.dag.old_job_id + ) + return SubmitJobOutput(job_id=job_id, data=dict(model_id="", model_version="")) + + @classmethod + def query_job(cls, query_job_input: QueryJobInput): + jobs = query_job_input.jobs + for job in jobs: + job.f_status = cls.update_status(job.f_status) + return QueryJobOutput(jobs=jobs) + + @classmethod + def stop_job(cls, stop_job_input: StopJobInput): + response = BfiaJobController.request_stop_job(stop_job_input.job_id) + cls.update_response(response) + return StopJobOutput(**response) + + @classmethod + def query_task(cls, query_task_input: QueryTaskInput): + tasks = query_task_input.tasks + for task in tasks: + task.f_status = cls.update_status(task.f_status) + task.f_party_status = cls.update_status(task.f_party_status) + return QueryTaskOutput(tasks=tasks) + + @staticmethod + def update_status(status): + RULES = { + BfiaJobStatus.PENDING: JobStatus.WAITING, + BfiaJobStatus.READY: JobStatus.WAITING, + BfiaJobStatus.RUNNING: JobStatus.RUNNING, + BfiaJobStatus.FINISHED: JobStatus.SUCCESS, + BfiaJobStatus.REJECTED: JobStatus.FAILED, + BfiaJobStatus.SUCCESS: JobStatus.SUCCESS, + BfiaJobStatus.FAILED: JobStatus.FAILED, + + } + return RULES[status] + + @staticmethod + def update_response(response): + message = response.pop("msg") + if message: + response["message"] = message + return response + diff --git a/python/fate_flow/adapter/bfia/conf/route_table.yaml b/python/fate_flow/adapter/bfia/conf/route_table.yaml new file mode 100644 index 000000000..0b02cf4ea --- /dev/null +++ b/python/fate_flow/adapter/bfia/conf/route_table.yaml @@ -0,0 +1,6 @@ +JG0100001100000000: + host: 127.0.0.1 + port: 9380 +JG0100001100000010: + host: 127.0.0.1 + port: 9380 diff --git a/python/fate_flow/scheduling_apps/__init__.py b/python/fate_flow/adapter/bfia/container/__init__.py similarity index 99% rename from python/fate_flow/scheduling_apps/__init__.py rename to python/fate_flow/adapter/bfia/container/__init__.py index 878d3a9c5..ae946a49c 100644 --- a/python/fate_flow/scheduling_apps/__init__.py +++ b/python/fate_flow/adapter/bfia/container/__init__.py @@ -12,4 +12,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# diff --git a/python/fate_flow/worker/__init__.py b/python/fate_flow/adapter/bfia/container/entrypoint/__init__.py similarity index 99% rename from python/fate_flow/worker/__init__.py rename to python/fate_flow/adapter/bfia/container/entrypoint/__init__.py index 878d3a9c5..ae946a49c 100644 --- a/python/fate_flow/worker/__init__.py +++ b/python/fate_flow/adapter/bfia/container/entrypoint/__init__.py @@ -12,4 +12,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# diff --git a/python/fate_flow/adapter/bfia/container/entrypoint/cli.py b/python/fate_flow/adapter/bfia/container/entrypoint/cli.py new file mode 100644 index 000000000..94bd9f812 --- /dev/null +++ b/python/fate_flow/adapter/bfia/container/entrypoint/cli.py @@ -0,0 +1,81 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import logging + +import click + +from fate_flow.adapter.bfia.container.wraps.wraps import BfiaWraps +from fate_flow.adapter.bfia.utils.spec.task import TaskRuntimeEnv + +logger = logging.getLogger(__name__) + + +@click.group() +def component(): + """ + Manipulate components: execute, list, generate describe file + """ + + +@component.command() +def entrypoint(): + configs = load_config_from_env() + logger = logging.getLogger(__name__) + logger.debug(f"task config: {configs}") + BfiaWraps(TaskRuntimeEnv(**configs)).run() + + +def load_config_from_env(): + import os + + config = {} + component_env_keys = ["system", "config", "runtime"] + sep = '.' + + for name in os.environ: + for key in component_env_keys: + if name.startswith(f"{key}{sep}"): + conf = os.environ[name] + try: + conf = json.loads(os.environ[name]) + except: + pass + config[name] = conf + + return unflatten_dict(config) + + +def unflatten_dict(flat_dict, sep='.'): + nested_dict = {} + + for key, value in flat_dict.items(): + keys = key.split(sep) + temp_dict = nested_dict + + for k in keys[:-1]: + if k not in temp_dict: + temp_dict[k] = {} + temp_dict = temp_dict[k] + + last_key = keys[-1] + if last_key in temp_dict and not isinstance(temp_dict[last_key], dict): + temp_dict[last_key] = {last_key: temp_dict[last_key]} + if last_key in temp_dict: + temp_dict[last_key].update({last_key: value}) + else: + temp_dict[last_key] = value + + return nested_dict diff --git a/python/fate_flow/adapter/bfia/container/entrypoint/runner.py b/python/fate_flow/adapter/bfia/container/entrypoint/runner.py new file mode 100644 index 000000000..316deb1e1 --- /dev/null +++ b/python/fate_flow/adapter/bfia/container/entrypoint/runner.py @@ -0,0 +1,29 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +class Submit: + @staticmethod + def run(): + import click + from fate_flow.adapter.bfia.container.entrypoint.cli import component + + cli = click.Group() + cli.add_command(component) + cli(prog_name="python -m fate_flow.adapter.bfia.container.entrypoint") + + +if __name__ == "__main__": + Submit.run() diff --git a/python/fate_flow/adapter/bfia/container/wraps/__init__.py b/python/fate_flow/adapter/bfia/container/wraps/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/adapter/bfia/container/wraps/wraps.py b/python/fate_flow/adapter/bfia/container/wraps/wraps.py new file mode 100644 index 000000000..7ab268c04 --- /dev/null +++ b/python/fate_flow/adapter/bfia/container/wraps/wraps.py @@ -0,0 +1,515 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import logging +import os.path +import subprocess +import sys +import traceback +from urllib.parse import urlparse + +import yaml + +from fate_flow.adapter.bfia.settings import FATE_CONTAINER_HOME +from fate_flow.adapter.bfia.utils.entity.status import TaskStatus +from fate_flow.adapter.bfia.utils.spec.task import TaskRuntimeEnv +from fate_flow.engine.backend import build_backend +from fate_flow.entity.code import ReturnCode +from fate_flow.entity.spec.dag import TaskConfigSpec, ArtifactInputApplySpec, Metadata, ArtifactOutputApplySpec, \ + ComponentSpecV1, TaskRuntimeConfSpec, FlowLogger, StandaloneComputingSpec, OSXFederationSpec, ComponentOutputMeta, \ + ArtifactOutputSpec +from fate_flow.entity.types import ProviderName +from fate_flow.hub.components_wraps import WrapsABC +from fate_flow.manager.worker.fate_executor import FateSubmit +from fate_flow.runtime.system_settings import DEFAULT_OUTPUT_DATA_PARTITIONS +from ofx.api.client import BfiaSchedulerApi + +logger = logging.getLogger(__name__) + +COMPUTING_ENGINE = "standalone" + + +class BfiaWraps(WrapsABC): + def __init__(self, config: TaskRuntimeEnv): + self.config = config + self.stages = "" + self.backend = build_backend(backend_name=COMPUTING_ENGINE) + self.io = DataIo(self.config.system.storage) + self._partitions = DEFAULT_OUTPUT_DATA_PARTITIONS + self._component_desc = None + self._output_map = {} + + @property + def mlmd(self): + if self.config.system.callback: + parsed_url = urlparse(self.config.system.callback) + client = BfiaSchedulerApi( + host=parsed_url.hostname, + port=parsed_url.port) + return client.worker + + @property + def task_info(self): + return { + "job_id": "job", + "role": self.self_role, + "party_id": self.self_party_id, + "task_id": self.config.config.task_id, + "task_version": 0 + } + + @property + def task_input_dir(self): + task_id = self.config.config.task_id + base_dir = FATE_CONTAINER_HOME + if self.config.config.log and self.config.config.log.path: + base_dir = self.config.config.log.path + path = os.path.join(base_dir, "jobs", task_id, self.self_role, "input") + os.makedirs(path, exist_ok=True) + return path + + @property + def task_output_dir(self): + task_id = self.config.config.task_id + base_dir = FATE_CONTAINER_HOME + if self.config.config.log and self.config.config.log.path: + base_dir = self.config.config.log.path + path = os.path.join(base_dir, "jobs", task_id, self.self_role, "output") + os.makedirs(path, exist_ok=True) + return path + + @property + def component_desc(self) -> ComponentSpecV1: + if not self._component_desc: + self.set_component_desc() + return self._component_desc + + @property + def self_role(self): + return self.config.config.self_role.split(".")[0] + + @property + def self_party_id(self): + role, party_index = self.config.config.self_role.split(".") + return self.config.config.node_id[role][party_index] + + @property + def data_home(self): + path = os.path.join(FATE_CONTAINER_HOME, "data") + os.makedirs(path, exist_ok=True) + return path + + @property + def parties(self): + parties_info = [] + for role, infos in self.config.config.node_id.items(): + for index, party_id in infos.items(): + parties_info.append({"role": role, "partyid": party_id}) + parties = { + "local": {"role": self.self_role, "partyid": self.self_party_id}, + "parties": parties_info + } + return parties + + @property + def generate_conf(self): + return TaskRuntimeConfSpec( + logger=self._generate_logger_conf(), + device=self._generate_device(), + computing=self._generate_computing_conf(), + federation=self._generate_federation_conf(), + storage=self.generate_storage_conf() + ) + + def task_logs_dir(self, *args): + task_id = self.config.config.task_id + base_dir = FATE_CONTAINER_HOME + if self.config.config.log and self.config.config.log.path: + base_dir = self.config.config.log.path + path = os.path.join(base_dir, "logs",task_id, *args) + os.makedirs(path, exist_ok=True) + return path + + def cleanup(self): + pass + + def run(self): + code = 0 + try: + config = self.preprocess() + output_meta = self.run_component(config) + self.push_output(output_meta) + print(output_meta) + code = output_meta.status.code + if output_meta.status.code != ReturnCode.Base.SUCCESS: + code = ReturnCode.Task.COMPONENT_RUN_FAILED + exceptions = output_meta.status.exceptions + logger.error(exceptions) + except Exception as e: + traceback.format_exc() + code = ReturnCode.Task.TASK_RUN_FAILED + print(e) + logger.error(e) + finally: + print(f"finish task with code {code}") + self.report_status(code) + if code: + sys.exit(code) + + def preprocess(self): + # set log + print("start preprocess") + self._generate_logger_conf().install() + logger = logging.getLogger(__name__) + + # input + logger.info("start generating input artifacts") + logger.info(self.config) + input_artifacts = self._preprocess_input_artifacts() + logger.info("input artifacts are ready") + logger.debug(input_artifacts) + logger.info(f"PYTHON PATH: {os.environ.get('PYTHONPATH')}") + + output_artifacts = self._preprocess_output_artifacts() + + logger.info(f"output_artifacts: {output_artifacts}") + config = TaskConfigSpec( + job_id="", + task_id=self.config.config.task_id, + party_task_id=f"{self.config.config.task_id}_{self.self_role}", + component=self.config.runtime.component.name, + role=self.self_role, + party_id=self.self_party_id, + stage=self.stages, + parameters=self.config.runtime.component.parameter, + input_artifacts=input_artifacts, + output_artifacts=output_artifacts, + conf=self.generate_conf, + task_name=self.config.runtime.component.name + ) + logger.debug(config) + print(config) + return config + + def run_component(self, config): + print("start run task") + task_parameters = config.dict() + logger.info("start run task") + os.makedirs(self.task_input_dir, exist_ok=True) + os.makedirs(self.task_output_dir, exist_ok=True) + conf_path = os.path.join(self.task_input_dir, "task_parameters.yaml") + task_result = os.path.join(self.task_output_dir, "task_result.yaml") + with open(conf_path, "w") as f: + yaml.dump(task_parameters, f) + code = self.backend.run( + provider_name=ProviderName.FATE, + task_info=self.task_info, + engine_run={"cores": 4}, + run_parameters=task_parameters, + output_path=task_result, + conf_path=conf_path, + sync=True, + config_dir=self.task_output_dir, std_dir=self.task_output_dir + ) + logger.info(f"finish task with code {code}") + print(f"finish task with code {code}") + + if os.path.exists(task_result): + with open(task_result, "r") as f: + try: + result = json.load(f) + output_meta = ComponentOutputMeta.parse_obj(result) + if code != 0: + output_meta.status.code = code + logger.debug(output_meta) + except: + raise RuntimeError(f"Task run failed, you can see the task result file for details: {task_result}") + else: + output_meta = ComponentOutputMeta(status=ComponentOutputMeta.Status( + code=ReturnCode.Task.NO_FOUND_RUN_RESULT, + exceptions=f"No found task output." + )) + return output_meta + + def push_output(self, output_meta: ComponentOutputMeta): + if self.task_end_with_success(output_meta.status.code): + if not output_meta.io_meta: + logger.info("No found io meta, pass push") + return + for key, datas in output_meta.io_meta.outputs.data.items(): + self._push_data(key, ArtifactOutputSpec(**datas)) + + # push model + for key, models in output_meta.io_meta.outputs.model.items(): + self._push_model(key, ArtifactOutputSpec(**models)) + + # push metric + for key, metrics in output_meta.io_meta.outputs.metric.items(): + self._push_metric(key, ArtifactOutputSpec(**metrics)) + + def set_component_desc(self): + component_ref = self.config.runtime.component.name + desc_file = os.path.join(self.task_output_dir, "define.yaml") + module_file_path = sys.modules[FateSubmit.__module__].__file__ + process_cmd = [ + sys.executable, + module_file_path, + "component", + "desc", + "--name", + component_ref, + "--save", + desc_file + ] + std = open(os.path.join(self.task_output_dir, "std.txt"), "w") + p = subprocess.Popen( + process_cmd, + env=os.environ, + stdout=std, + stderr=std + ) + p.wait() + if os.path.exists(desc_file): + with open(desc_file, "r") as fr: + desc = yaml.safe_load(fr) + self._component_desc = ComponentSpecV1(**desc) + else: + raise RuntimeError("No found description file") + + def _preprocess_output_artifacts(self): + output_map = {} + output_artifacts = {} + if self.config.runtime.component.output: + for artifacts_type, artifact in self.component_desc.component.output_artifacts.dict().items(): + for key in self.config.runtime.component.output: + if key in artifact: + uri = f"file://{self.task_output_dir}/{key}" + if artifacts_type == "data": + address = self.config.runtime.component.output.get(key) + uri = f"standalone://{self.data_home}/{address.namespace}/{address.name}" + + if self.config.runtime.component.name == "dataframe_transformer": + uri = f"standalone://{self.data_home}/" \ + f"{self.config.runtime.component.parameter.get('namespace')}/" \ + f"{self.config.runtime.component.parameter.get('name')}" + + output_artifact = ArtifactOutputApplySpec( + uri=uri, + type_name=artifact[key]["types"][0] + ) + output_map[key] = artifacts_type + output_artifacts[key] = output_artifact + self._output_map = output_map + return output_artifacts + + def _preprocess_input_artifacts(self): + input_artifacts = {} + + os.environ["STANDALONE_DATA_PATH"] = self.data_home + + # only data + data = self.component_desc.component.input_artifacts.data + stage = "" + if self.config.runtime.component.input: + for name, address in self.config.runtime.component.input.items(): + if name in data: + # set stage + stage = data[name].stages[0] + if self.self_role not in data[name].roles: + logger.info(f"role {self.self_role} does not rely on data {name} input") + continue + path = os.path.join(self.data_home, address.namespace, address.name) + os.makedirs(os.path.dirname(path), exist_ok=True) + metadata = self.io.s3_to_local(address, self.data_home, path=path) + input_artifacts[name] = metadata + self._set_stage(stage) + return input_artifacts + + def _set_stage(self, stage): + self.stages = self.config.runtime.component.parameter.pop("stage", None) or stage or "train" + + def _generate_logger_conf(self): + level = "DEBUG" + delay = True + formatters = None + return FlowLogger.create( + task_log_dir=self.task_logs_dir(), + job_party_log_dir=self.task_logs_dir(self.self_role), + level=level, + delay=delay, + formatters=formatters + ) + + @staticmethod + def _generate_device(): + return { + "type": "CPU" + } + + def _generate_computing_conf(self): + return StandaloneComputingSpec( + type=COMPUTING_ENGINE, + metadata=dict( + computing_id=f"{self.config.config.session_id}_{self.self_role}", + options=dict(data_dir=self.data_home) + ) + ) + + def _generate_federation_conf(self): + host, port = self.config.system.transport.split(":") + return OSXFederationSpec(type="osx", metadata=OSXFederationSpec.MetadataSpec( + federation_id=self.config.config.session_id, + parties=self.parties, + osx_config=OSXFederationSpec.MetadataSpec.OSXConfig(host=host, port=int(port)) + )) + + @staticmethod + def generate_storage_conf(): + return COMPUTING_ENGINE + + @staticmethod + def task_end_with_success(code): + return code == 0 + + def _push_data(self, output_key, output_data: ArtifactOutputSpec): + if output_data.consumed is False: + return + logger.info("save data") + meta = ArtifactInputApplySpec( + metadata=Metadata( + metadata=dict(options=dict(partitions=self._partitions)) + ), + uri="" + ) + meta.metadata.metadata["schema"] = output_data.metadata.metadata.get("schema", {}) + meta.metadata.source = output_data.metadata.source + address = self.config.runtime.component.output.get(output_key) + if output_data.metadata.name and output_data.metadata.namespace: + name, namespace = output_data.metadata.name, output_data.metadata.namespace + else: + name, namespace = address.name, address.namespace + path = output_data.uri.split("://")[1] + logger.info(f"start upload {path} to s3") + logger.info(f"namespace {namespace} name {name}") + self.io.upload_to_s3(path, name, namespace, metadata=meta.metadata.dict()) + + def _push_model(self, output_key, output_model: ArtifactOutputSpec): + address = self.config.runtime.component.output.get(output_key) + logger.info("save model") + meta = ArtifactInputApplySpec( + metadata=Metadata( + metadata=dict() + ), + uri="" + ) + meta.metadata.model_key = output_model.metadata.model_key + meta.metadata.source = output_model.metadata.source + meta.metadata.model_overview = output_model.metadata.model_overview + meta.metadata.type_name = output_model.metadata.type_name + self.io.upload_to_s3(output_model.uri.split("://")[1], address.name, address.namespace, metadata=meta.dict()) + + def _push_metric(self, output_key, output_metric: ArtifactOutputSpec): + address = self.config.runtime.component.output.get(output_key) + logger.info("save metric") + meta = ArtifactInputApplySpec( + metadata=Metadata( + metadata=dict() + ), + uri="" + ) + meta.metadata.source = output_metric.metadata.source + meta.metadata.type_name = output_metric.metadata.type_name + self.io.upload_to_s3(output_metric.uri.split("://")[1], address.name, address.namespace, metadata=meta.dict()) + + def report_status(self, code): + if self.task_end_with_success(code): + resp = self.mlmd.report_task_status( + task_id=self.config.config.task_id, + role=self.config.config.self_role, + status=TaskStatus.SUCCESS + ) + else: + resp = self.mlmd.report_task_status( + task_id=self.config.config.task_id, + role=self.config.config.self_role, + status=TaskStatus.FAILED + ) + self.log_response(resp, req_info="report status") + + @staticmethod + def log_response(resp, req_info): + try: + logger.info(resp.json()) + resp_json = resp.json() + if resp_json.get("code") != ReturnCode.Base.SUCCESS: + logging.exception(f"{req_info}: {resp.text}") + except Exception: + logger.error(f"{req_info}: {resp.text}") + + +class DataIo(object): + def __init__(self, s3_address): + self.s3_address = s3_address + self._storage_session = None + + @property + def storage_session(self): + if not self._storage_session: + protocol, host, port, parameters = self.parser_storage_address(self.s3_address) + from fate_flow.adapter.bfia import engine_storage + session = engine_storage.session.S3Session(url=f"http://{host}:{port}", **parameters) + self._storage_session = session + return self._storage_session + + @staticmethod + def parser_storage_address(storage_address): + from urllib.parse import urlparse, parse_qs + # url = "s3://ip:port?name=xxx&password=xxx" + parsed_url = urlparse(storage_address) + protocol = parsed_url.scheme + host = parsed_url.hostname + port = parsed_url.port + + ps = parse_qs(parsed_url.query) + parameters = {} + for key in ps.keys(): + parameters[key] = ps.get(key, [''])[0] + return protocol, host, port, parameters + + def s3_to_local(self, address, data_home, path): + table = self.storage_session.get_table(name=address.name, namespace=address.namespace) + table.download_data_to_local(local_path=path) + schema = json.loads(table.meta_output()).get("metadata") + if not schema: + schema = {} + logger.debug(json.dumps(schema)) + metadata = Metadata(**schema) + self._partitions = metadata.metadata.get("options", {}).get("partitions", DEFAULT_OUTPUT_DATA_PARTITIONS) + meta = ArtifactInputApplySpec( + metadata=Metadata( + **schema + ), + uri=f"{COMPUTING_ENGINE}:///{address.namespace}/{address.name}" + ) + from fate.arch.computing.backends.standalone import standalone_raw + standalone_raw._TableMetaManager.add_table_meta( + data_dir=data_home, namespace=address.namespace, name=address.name, num_partitions=self._partitions, + key_serdes_type=0, value_serdes_type=0, partitioner_type=0 + ) + return meta + + def upload_to_s3(self, path, name, namespace, metadata): + table = self.storage_session.create_table(name, namespace, column_info=[], metadata=metadata) + table.upload_local_data(path) diff --git a/python/fate_flow/adapter/bfia/db/__init__.py b/python/fate_flow/adapter/bfia/db/__init__.py new file mode 100644 index 000000000..d901f11d9 --- /dev/null +++ b/python/fate_flow/adapter/bfia/db/__init__.py @@ -0,0 +1,3 @@ +__all__ = ["ComponentOutput"] + +from fate_flow.adapter.bfia.db.output import ComponentOutput diff --git a/python/fate_flow/adapter/bfia/db/output.py b/python/fate_flow/adapter/bfia/db/output.py new file mode 100644 index 000000000..b555d8b53 --- /dev/null +++ b/python/fate_flow/adapter/bfia/db/output.py @@ -0,0 +1,20 @@ +from peewee import CompositeKey, CharField, TextField, BigIntegerField + +from fate_flow.db import DataBaseModel, JSONField + + +class ComponentOutput(DataBaseModel): + f_job_id = CharField(max_length=25, index=True) + f_role = CharField(max_length=50, index=True) + f_node_id = CharField(max_length=50, index=True) + f_task_name = CharField(max_length=50) + f_component = CharField(max_length=50) + f_task_id = CharField(max_length=100) + f_type = CharField(max_length=20) + f_key = CharField(max_length=20) + f_engine = JSONField() + f_address = JSONField() + + class Meta: + db_table = "t_bfia_component_output" + primary_key = CompositeKey('f_job_id', 'f_role', 'f_node_id', 'f_task_id', 'f_type', 'f_key') diff --git a/python/fate_flow/adapter/bfia/engine_storage/__init__.py b/python/fate_flow/adapter/bfia/engine_storage/__init__.py new file mode 100644 index 000000000..2a98f7a0f --- /dev/null +++ b/python/fate_flow/adapter/bfia/engine_storage/__init__.py @@ -0,0 +1,29 @@ +import os +from urllib.parse import urlparse + +from .session import S3Session + + +def inti_session(): + """创建客户端对象""" + + storage_str = os.getenv('system.storage') + assert storage_str is not None, 'system.storage 配置为空' + + storage_str = urlparse(storage_str) + storage_query = {i.split('=')[0]: i.split('=')[1] for i in storage_str.query.split('&')} + username = storage_query.get('username') + password = storage_query.get('password') + url = storage_str.netloc + service_type = storage_str.scheme + + assert url is not None, 'system.storage 配置无法解析url' + assert username is not None, 'system.storage 配置无法解析username' + assert password is not None, 'system.storage 配置无法解析password' + assert service_type is not None, 'system.storage 配置无法解析service_type' + + if service_type == 's3': + url = f"http://{url}" + return S3Session(url=url, username=username, password=password) + else: + assert Exception(f'不支持相应配置, storage_service_type: {service_type}') diff --git a/python/fate_flow/adapter/bfia/engine_storage/client.py b/python/fate_flow/adapter/bfia/engine_storage/client.py new file mode 100644 index 000000000..e11b1e89d --- /dev/null +++ b/python/fate_flow/adapter/bfia/engine_storage/client.py @@ -0,0 +1,58 @@ +import boto3 + + +class S3Client(object): + + def __init__(self, url: str, username: str, password: str): + self.url = url + self.username = username + self.password = password + + def list_buckets(self): + session = boto3.Session(aws_access_key_id=self.username, aws_secret_access_key=self.password) + s3_client = session.client(service_name="s3", endpoint_url=self.url) + return s3_client.list_buckets() + + def create_bucket(self, bucket_name): + session = boto3.Session(aws_access_key_id=self.username, aws_secret_access_key=self.password) + s3_client = session.client(service_name="s3", endpoint_url=self.url) + s3_client.create_bucket(Bucket=bucket_name) + + def put_object(self, bucket: str, key: str, body: bytes): + session = boto3.Session(aws_access_key_id=self.username, aws_secret_access_key=self.password) + s3_client = session.client(service_name="s3", endpoint_url=self.url) + s3_client.put_object(Bucket=bucket, Key=key, Body=body) + + def get_object(self, bucket: str, key: str): + session = boto3.Session(aws_access_key_id=self.username, aws_secret_access_key=self.password) + s3_client = session.client(service_name="s3", endpoint_url=self.url) + return s3_client.get_object(Bucket=bucket, Key=key) + + def head_object(self, bucket: str, key: str): + session = boto3.Session(aws_access_key_id=self.username, aws_secret_access_key=self.password) + s3_client = session.client(service_name="s3", endpoint_url=self.url) + + return s3_client.head_object(Bucket=bucket, Key=key) + + def object_exist(self, bucket: str, key: str): + try: + self.head_object(bucket=bucket, key=key) + return True + except Exception as e: + return False + + def upload_file(self, file_path: str, bucket: str, key: str): + session = boto3.Session(aws_access_key_id=self.username, aws_secret_access_key=self.password) + s3_client = session.client(service_name="s3", endpoint_url=self.url) + s3_client.upload_file(Bucket=bucket, Key=key, Filename=file_path) + + def download_file(self, file_path: str, bucket: str, key: str): + session = boto3.Session(aws_access_key_id=self.username, aws_secret_access_key=self.password) + s3_client = session.client(service_name="s3", endpoint_url=self.url) + s3_client.download_file(Bucket=bucket, Key=key, Filename=file_path) + + def delete_folder(self, bucket: str, key: str): + s3 = boto3.resource(service_name='s3', endpoint_url=self.url,use_ssl=False, aws_access_key_id=self.username, + aws_secret_access_key=self.password) + bucket = s3.Bucket(bucket) + bucket.objects.filter(Prefix=key).delete() diff --git a/python/fate_flow/adapter/bfia/engine_storage/consts.py b/python/fate_flow/adapter/bfia/engine_storage/consts.py new file mode 100644 index 000000000..cf698f877 --- /dev/null +++ b/python/fate_flow/adapter/bfia/engine_storage/consts.py @@ -0,0 +1,4 @@ +S3_BUCKET_NAME = 'storage' +S3_META_KEY = '{namespace}/{name}/metadata' +S3_OBJECT_KEY = '{namespace}/{name}/data_{partition}' +S3_SAVE_PATH = '/{namespace}/{name}' diff --git a/python/fate_flow/adapter/bfia/engine_storage/meta.py b/python/fate_flow/adapter/bfia/engine_storage/meta.py new file mode 100644 index 000000000..3faddbd90 --- /dev/null +++ b/python/fate_flow/adapter/bfia/engine_storage/meta.py @@ -0,0 +1,66 @@ +import datetime +import json + + +class ColumnInfo(dict): + """特征信息""" + name = '' # 特征名称 + type = '' # 特征类型 + + def __init__(self, name: str, type: str): + super().__init__() + self.name = name + self.type = type + + def __getattr__(self, key): + return self[key] + + def __setattr__(self, key, value): + self[key] = value + + +class Meta(object): + count = 0 # 数据量 + partitions = 1 # 分区数 + column_info = [] # 特征信息 + description = '' # 数据描述 + create_time = datetime.datetime.now() # 数据创建时间 + expire_time = create_time + datetime.timedelta(days=365) # 数据有效截至时间 + file_type = '' # 数据集文件类型 + data_type = 'dense' # 数据集类型 + metadata = {} + + def __init__(self, column_info: list, options: str, partition: int, description: str, metadata: dict): + self.column_info = column_info + self.options = options + self.partitions = partition + self.description = description + self.metadata = metadata + + def meta_output(self): + return json.dumps({ + "column_info": self.column_info, + "options": self.options, + "partitions": self.partitions, + "description": self.description, + "data_type": self.data_type, + "file_type": self.file_type, + "create_time": self.create_time.timestamp(), + "expire_time": self.expire_time.timestamp(), + "count": self.count, + "metadata": self.metadata + }) + + def meta_input(self, meta_data): + meta_json = json.loads(meta_data) + self.column_info = meta_json.get('column_info') + self.column_info = [ColumnInfo(**c) for c in self.column_info] # 转成对象 + self.options = meta_json.get('options') + self.partitions = meta_json.get('partitions') + self.description = meta_json.get('description') + self.count = meta_json.get('count') + self.create_time = datetime.datetime.fromtimestamp(meta_json.get('create_time')) + self.expire_time = datetime.datetime.fromtimestamp(meta_json.get('expire_time')) + self.file_type = meta_json.get('file_type') + self.data_type = meta_json.get('data_type') + self.metadata = meta_json.get("metadata") diff --git a/python/fate_flow/adapter/bfia/engine_storage/session.py b/python/fate_flow/adapter/bfia/engine_storage/session.py new file mode 100644 index 000000000..3fef7e4ce --- /dev/null +++ b/python/fate_flow/adapter/bfia/engine_storage/session.py @@ -0,0 +1,85 @@ + +from .client import S3Client +from .consts import S3_BUCKET_NAME, S3_META_KEY, S3_OBJECT_KEY, S3_SAVE_PATH +from .table import S3Table, FateTable + + +class Session(object): + def __init__(self, url: str, username: str, password: str): + pass + + def create_table(self, name: str, namespace: str, column_info: list = None, partition: int = 1, + description: str = ''): + """创建table""" + pass + + def get_table(self, name: str, namespace: str): + """获取table""" + pass + + def delete_table(self, name: str, namespace: str): + """删除table""" + pass + + def is_exist(self, name: str, namespace: str): + """判断数据是否存在""" + pass + + +class S3Session(Session): + """s3服务连接实现""" + def __init__(self, url: str, username: str, password: str): + super(S3Session, self).__init__(url, username, password) + + self.url = url + self.username = username + self.password = password + + self._create_bucket() + + def _create_bucket(self): + """创建仓库""" + client = S3Client(url=self.url, username=self.username, password=self.password) + response = client.list_buckets() + bucket_names = [b.get('Name') for b in response.get('Buckets', [])] + if S3_BUCKET_NAME not in bucket_names: + client.create_bucket(S3_BUCKET_NAME) + + def create_table(self, name: str, namespace: str, column_info: list = None, partition: int = 1, + description: str = '', metadata: dict = {}): + """创建table""" + if partition != 1: + raise Exception('暂不支持partition分区') + table = FateTable(url=self.url, username=self.username, password=self.password, name=name, namespace=namespace, + column_info=column_info, partition=partition, description=description, metadata=metadata) + client = S3Client(url=self.url, username=self.username, password=self.password) + client.put_object(bucket=S3_BUCKET_NAME, key=S3_META_KEY.format(namespace=namespace, name=name), + body=table.meta_output()) + return table + + def get_table(self, name: str, namespace: str): + """获取table""" + client = S3Client(url=self.url, username=self.username, password=self.password) + + # 检查table是否存在 + if not client.object_exist(bucket=S3_BUCKET_NAME, key=S3_META_KEY.format(namespace=namespace, name=name)): + raise ValueError('table不存在，请检查参数配置') + + # 创建空table，并读入meta数据 + meta_response = client.get_object(bucket=S3_BUCKET_NAME, key=S3_META_KEY.format(namespace=namespace, name=name)) + table = FateTable(name=name, namespace=namespace, url=self.url, username=self.username, password=self.password) + try: + meta_data = meta_response.get('Body').read() + except Exception as e: + raise Exception('读取meta数据失败') + table.meta_input(meta_data) + return table + + def delete_table(self, name: str, namespace: str): + client = S3Client(url=self.url, username=self.username, password=self.password) + key = S3_SAVE_PATH.format(namespace=namespace, name=name) + client.delete_folder(bucket=S3_BUCKET_NAME, key=key) + + def is_exist(self, name: str, namespace: str): + client = S3Client(url=self.url, username=self.username, password=self.password) + return client.object_exist(bucket=S3_BUCKET_NAME, key=S3_META_KEY.format(namespace=namespace, name=name)) diff --git a/python/fate_flow/adapter/bfia/engine_storage/table.py b/python/fate_flow/adapter/bfia/engine_storage/table.py new file mode 100644 index 000000000..23eda132c --- /dev/null +++ b/python/fate_flow/adapter/bfia/engine_storage/table.py @@ -0,0 +1,198 @@ +import os +import tarfile +import uuid +from tempfile import TemporaryDirectory + +from .client import S3Client +from .consts import S3_BUCKET_NAME, S3_OBJECT_KEY, S3_SAVE_PATH, S3_META_KEY +from .meta import Meta, ColumnInfo + + +class Table(Meta): + """数据实体""" + name = '' # 数据表名称 + namespace = '' # 数据表明明空间 + + def __init__(self, name: str, namespace: str, column_info: list = None, partition: int = 1, options: str = '', + description: str = '', metadata={}): + super().__init__(column_info=column_info, partition=partition, options=options, description=description, metadata=metadata) + self.name = name + self.namespace = namespace + + def upload(self, local_path: str, overwrite: bool = True, callback_func=None, file_type: str = 'csv'): + """上传本地文件""" + pass + + def download(self, local_path: str, overwrite: bool = True, callback_func=None): + """下载文件""" + pass + + def read(self, partition_index: int = 1, callback_func=None) -> bytes: + """读取文件""" + pass + + def write(self, data: bytes, partition_id: int = 1, overwrite: bool = True, callback_func=None, + file_type: str = 'csv'): + """写文件""" + pass + + def set_column_info(self, column_info: list): + """设置特征信息""" + pass + + def get_column_info(self) -> list: + """获取特征信息""" + pass + + def set_description(self, description: str): + """设置description""" + pass + + def get_description(self) -> str: + """获取description""" + pass + + def get_partitions(self) -> list: + """获取分区集合""" + pass + + +class S3Table(Table): + + def __init__(self, **kwargs): + self.s3_url = kwargs.pop('url') + self.s3_username = kwargs.pop('username') + self.s3_password = kwargs.pop('password') + super().__init__(**kwargs) + + def upload(self, local_path: str, overwrite: bool = True, callback_func=None, file_type: str = 'csv'): + self.file_type = file_type + + client = S3Client(url=self.s3_url, username=self.s3_username, password=self.s3_password) + if not os.path.exists(local_path): + raise ValueError('路径文件不存在') + # if os.path.isdir(local_path): + # raise ValueError('暂不支持文件夹') + + # # 统计行数 + # with open(local_path, 'r') as f: + # lines = f.readlines() + # self.count = len(lines) - 1 # 去除首行 + + # TODO: 分区暂固定为0 + p_idx = 0 + # 是否覆盖 + key = S3_OBJECT_KEY.format(namespace=self.namespace, name=self.name, partition=p_idx) + object_exist = client.object_exist(bucket=S3_BUCKET_NAME, key=key) + if object_exist and not overwrite: + raise ValueError(f'namespace:{self.namespace}, name:{self.name} object已存在，如需覆盖请配置overwrite为True') + + client.upload_file(file_path=local_path, bucket=S3_BUCKET_NAME, + key=S3_OBJECT_KEY.format(namespace=self.namespace, name=self.name, partition=p_idx)) + + if callback_func is not None: + callback_func() + + def download(self, local_path: str, overwrite: bool = True, callback_func=None): + if os.path.exists(local_path) and not overwrite: + raise ValueError(f'已存在文件{local_path}, 如需覆盖请配置overwrite为True') + client = S3Client(url=self.s3_url, username=self.s3_username, password=self.s3_password) + # TODO: 分区暂固定为0 + p_idx = 0 + client.download_file(file_path=local_path, bucket=S3_BUCKET_NAME, key=S3_OBJECT_KEY.format(namespace=self.namespace, name=self.name, partition=p_idx)) + + if callback_func is not None: + callback_func() + + def read(self, partition_index: int = 1, callback_func=None) -> bytes: + # TODO: 分区暂固定为0 + partition_index = 0 + client = S3Client(url=self.s3_url, username=self.s3_username, password=self.s3_password) + key = S3_OBJECT_KEY.format(namespace=self.namespace, name=self.name, partition=partition_index) + + if not client.object_exist(bucket=S3_BUCKET_NAME, key=key): + raise ValueError(f'namespace:{self.namespace}, name:{self.name} object不存在') + + response = client.get_object(bucket=S3_BUCKET_NAME, key=key) + data = response.get('Body').read() + if callback_func: + callback_func() + return data + + def write(self, data: bytes, partition_id: int = 1, overwrite: bool = True, callback_func=None, + file_type: str = 'csv'): + self.file_type = file_type + if data.endswith(b'\n'): + self.count = data.count(b'\n') - 1 # 统计\n数目判断行数 + else: + self.count = data.count(b'\n') # 统计\n数目判断行数 + # TODO: 分区暂固定为0 + partition_index = 0 + client = S3Client(url=self.s3_url, username=self.s3_username, password=self.s3_password) + key = S3_OBJECT_KEY.format(namespace=self.namespace, name=self.name, partition=partition_index) + + object_exist = client.object_exist(bucket=S3_BUCKET_NAME, key=key) + if object_exist and not overwrite: + raise ValueError(f'namespace:{self.namespace}, name:{self.name} object已存在，如需覆盖请配置overwrite为True') + client.put_object(body=data, bucket=S3_BUCKET_NAME, key=key) + + def get_path(self) -> str: + """获取s3储存路径""" + return 's3://' + S3_BUCKET_NAME + S3_SAVE_PATH.format(namespace=self.namespace, name=self.name) + + def set_column_info(self, column_info: list): + for c in column_info: + assert isinstance(c, ColumnInfo), 'column_info数据类型需为ColumnInfo' + + self.column_info = column_info + client = S3Client(url=self.s3_url, username=self.s3_username, password=self.s3_password) + client.put_object(bucket=S3_BUCKET_NAME, key=S3_META_KEY.format(namespace=self.namespace, name=self.name), + body=self.meta_output()) + + def get_column_info(self): + return self.column_info + + def set_description(self, description: str): + self.description = description + client = S3Client(url=self.s3_url, username=self.s3_username, password=self.s3_password) + client.put_object(bucket=S3_BUCKET_NAME, key=S3_META_KEY.format(namespace=self.namespace, name=self.name), + body=self.meta_output()) + + def get_description(self) -> str: + return self.description + + def get_partitions(self) -> list: + # TODO: 分区暂固定为0 + return ['data_0'] + + +class FateTable(S3Table): + def upload_local_data(self, path, overwrite: bool = True, callback_func=None, file_type: str = 'csv'): + with TemporaryDirectory() as output_tmp_dir: + if os.path.isdir(path): + temp_path = os.path.join(output_tmp_dir, str(uuid.uuid1())) + self._tar(path, temp_path) + path = temp_path + self.upload(path, overwrite, callback_func, file_type) + + def download_data_to_local(self, local_path): + with TemporaryDirectory() as output_tmp_dir: + temp_path = os.path.join(output_tmp_dir, str(uuid.uuid1())) + self.download(local_path=temp_path) + self._x_tar(temp_path, local_path) + + @staticmethod + def _tar(source_directory, target_archive): + with tarfile.open(fileobj=open(target_archive, "wb"), mode='w:gz') as tar: + for root, _, files in os.walk(source_directory): + for file in files: + full_path = os.path.join(root, file) + rel_path = os.path.relpath(full_path, source_directory) + tar.add(full_path, rel_path) + + @staticmethod + def _x_tar(path, download_path): + tar = tarfile.open(path, "r:gz") + file_names = tar.getnames() + for file_name in file_names: + tar.extract(file_name, download_path) diff --git a/python/fate_flow/adapter/bfia/examples/job/fate/fate_components.json b/python/fate_flow/adapter/bfia/examples/job/fate/fate_components.json new file mode 100644 index 000000000..a883156ad --- /dev/null +++ b/python/fate_flow/adapter/bfia/examples/job/fate/fate_components.json @@ -0,0 +1,524 @@ +{ + "name": "fate", + "device": "docker", + "version": "2.0.0", + "metadata": { + "base_url": "", + "image": "federatedai/standalone_fate:2.0.0.beta" + }, + "protocol": "bfia", + "components_description": { + "psi": { + "componentName": "psi", + "title": "对齐算法", + "provider": "fate", + "version": "2.0.0", + "description": "对齐算法", + "roleList": [ + "guest", + "host" + ], + "desVersion": "2.0.0", + "storageEngine": [ + "s3", + "hdfs", + "eggroll" + ], + "inputParam": [ + { + "name": "protocol", + "title": "protocol", + "description": "protocol", + "type": "string", + "optional": "true", + "defaultValue": "ecdh_psi", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "curve_type", + "title": "curve_type", + "description": "curve_type", + "type": "string", + "optional": "true", + "defaultValue": "curve25519", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [ + { + "name": "input_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + } + ], + "outputData": [ + { + "name": "output_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + }, + { + "name": "metric", + "description": "对齐数", + "category": "report", + "dataFormat": [ + "json" + ] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] + }, + "coordinated_lr": { + "componentName": "coordinated_lr", + "title": "lr", + "provider": "fate", + "version": "2.0.0", + "description": "逻辑回归算法", + "roleList": [ + "guest", + "host", + "arbiter" + ], + "desVersion": "2.0.0", + "storageEngine": [ + "s3", + "hdfs", + "eggroll" + ], + "inputParam": [ + { + "name": "learning_rate_scheduler", + "title": "", + "description": "learning rate scheduler", + "type": "string", + "optional": "true", + "defaultValue": "{'method': 'linear', 'scheduler_params': {'start_factor': 1.0}}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "epochs", + "title": "", + "description": "max iteration num", + "type": "integer", + "optional": "true", + "defaultValue": 20, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "batch_size", + "title": "", + "description": "batch size, None means full batch, otherwise should be no less than 10, default None", + "type": "integer", + "optional": "true", + "defaultValue": null, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "optimizer", + "title": "", + "description": "", + "type": "string", + "optional": "true", + "defaultValue": "{'method': 'sgd', 'penalty': 'l2', 'plpah': 1.0, 'optimizer_params': {'lr': 1e-2, 'weight_decay': 0}}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "floating_point_precision", + "title": "", + "description": "floating point precision", + "type": "integer", + "optional": "true", + "defaultValue": 23, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "tol", + "title": "", + "description": "", + "type": "float", + "optional": "true", + "defaultValue": 1e-4, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "early_stop", + "title": "", + "description": "early stopping criterion, choose from {weight_diff, diff, abs, val_metrics}", + "type": "string", + "optional": "true", + "defaultValue": "diff", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "he_param", + "title": "", + "description": "homomorphic encryption param", + "type": "string", + "optional": "true", + "defaultValue": "{'kind': 'paillier', 'key_length': 1024}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "init_param", + "title": "", + "description": "Model param init setting.", + "type": "string", + "optional": "true", + "defaultValue": "{'method': 'random_uniform', 'fit_intercept': true, 'random_state': null}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "threshold", + "title": "", + "description": "predict threshold for binary data", + "type": "float", + "optional": "true", + "defaultValue": 0.5, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + } + ], + "outputData": [ + { + "name": "train_output_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + }, + { + "name": "output_model", + "description": "模型", + "category": "model", + "dataFormat": [ + "json" + ] + }, + { + "name": "metric", + "description": "report", + "category": "report", + "dataFormat": [ + "json" + ] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] + }, + "hetero_secureboost": { + "componentName": "hetero_secureboost", + "title": "sbt", + "provider": "fate", + "version": "2.0.0", + "description": "XGBoost算法", + "roleList": [ + "guest", + "host" + ], + "desVersion": "2.0.0", + "storageEngine": [ + "s3", + "hdfs", + "eggroll" + ], + "inputParam": [ + { + "name": "num_trees", + "title": "", + "description": "", + "type": "integer", + "optional": "true", + "defaultValue": 3, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "learning_rate", + "title": "", + "description": "", + "type": "float", + "optional": "true", + "defaultValue": 0.3, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "max_depth", + "title": "", + "description": "max depth of a tree", + "type": "integer", + "optional": "true", + "defaultValue": 3, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "max_bin", + "title": "", + "description": "max bin number of feature binning", + "type": "integer", + "optional": "true", + "defaultValue": 32, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "objective", + "title": "", + "description": "", + "type": "string", + "optional": "true", + "defaultValue": "binary:bce", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "num_class", + "title": "", + "description": "", + "type": "integer", + "optional": "true", + "defaultValue": 2, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "l2", + "title": "", + "description": "L2 regularization", + "type": "float", + "optional": "true", + "defaultValue": 0.1, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_impurity_split", + "title": "", + "description": "min impurity when splitting a tree node", + "type": "float", + "optional": "true", + "defaultValue": 1e-2, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_sample_split", + "title": "", + "description": "min sample to split a tree node", + "type": "integer", + "optional": "true", + "defaultValue": 2, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_leaf_node", + "title": "", + "description": "mininum sample contained in a leaf node", + "type": "integer", + "optional": "true", + "defaultValue": 1, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_child_weight", + "title": "", + "description": "minumum hessian contained in a leaf node", + "type": "integer", + "optional": "true", + "defaultValue": 1, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "gh_pack", + "title": "", + "description": "whether to pack gradient and hessian together", + "type": "bool", + "optional": "true", + "defaultValue": true, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "split_info_pack", + "title": "", + "description": "for host side, whether to pack split info together", + "type": "bool", + "optional": "true", + "defaultValue": true, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "hist_sub", + "title": "", + "description": "whether to use histogram subtraction", + "type": "bool", + "optional": "true", + "defaultValue": true, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "he_param", + "title": "", + "description": "whether to use histogram subtraction", + "type": "string", + "optional": "true", + "defaultValue": "{'kind': 'paillier', 'key_length': 1024}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + } + ], + "outputData": [ + { + "name": "train_data_output", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + }, + { + "name": "train_model_output", + "description": "输出模型", + "category": "model", + "dataFormat": [ + "json" + ] + }, + { + "name": "metric", + "description": "对齐数", + "category": "report", + "dataFormat": [ + "json" + ] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] + } + } +} \ No newline at end of file diff --git a/python/fate_flow/adapter/bfia/examples/job/fate/fate_psi_lr.json b/python/fate_flow/adapter/bfia/examples/job/fate/fate_psi_lr.json new file mode 100644 index 000000000..896a1d382 --- /dev/null +++ b/python/fate_flow/adapter/bfia/examples/job/fate/fate_psi_lr.json @@ -0,0 +1,112 @@ +{ + "flow_id": "", + "dag": { + "components": [ + { + "name": "psi_0", + "componentName": "psi", + "provider": "fate", + "version": "2.0.0", + "input": [], + "output": [ + { + "type": "dataset", + "key": "output_data" + }, + { + "type": "report", + "key": "metric" + } + ] + }, + { + "name": "lr_0", + "componentName": "coordinated_lr", + "provider": "fate", + "version": "2.0.0", + "input": [{ + "type": "dataset", + "key": "psi_0.output_data" + }], + "output": [ + { + "type": "dataset", + "key": "train_output_data" + }, + { + "type": "model", + "key": "output_model" + }, + { + "type": "report", + "key": "metric" + } + ] + } + ], + "version": "2.0" + }, + "config": { + "initiator": { + "role": "guest", + "node_id": "JG0100001100000010" + }, + "role": { + "guest": [ + "JG0100001100000010" + ], + "host": [ + "JG0100001100000010" + ], + "arbiter": [ + "JG0100001100000010" + ] + }, + "job_params": { + "common": { + "sync_type": "poll" + }, + "guest": { + "0": { + "resources": { + "cpu": -1, + "memory": -1, + "disk": -1 + } + } + }, + "host": { + "0": { + "resources": { + "cpu": -1, + "memory": -1, + "disk": -1 + } + } + } + }, + "task_params": { + "common": { + "psi_0":{ + }, + "lr_0": { + } + }, + "guest": { + "0": { + "psi_0": { + "dataset_id": "test_data#guest" + } + } + }, + "host": { + "0": { + "psi_0": { + "dataset_id": "test_data#host" + } + } + } + }, + "version": "2.0.0" + } +} \ No newline at end of file diff --git a/python/fate_flow/adapter/bfia/examples/job/fate/fate_psi_sbt.json b/python/fate_flow/adapter/bfia/examples/job/fate/fate_psi_sbt.json new file mode 100644 index 000000000..6fee01da1 --- /dev/null +++ b/python/fate_flow/adapter/bfia/examples/job/fate/fate_psi_sbt.json @@ -0,0 +1,109 @@ +{ + "flow_id": "", + "dag": { + "components": [ + { + "name": "psi_0", + "componentName": "psi", + "provider": "fate", + "version": "2.0.0", + "input": [], + "output": [ + { + "type": "dataset", + "key": "output_data" + }, + { + "type": "report", + "key": "metric" + } + ] + }, + { + "name": "sbt_0", + "componentName": "hetero_secureboost", + "provider": "fate", + "version": "2.0.0", + "input": [{ + "type": "dataset", + "key": "psi_0.output_data" + }], + "output": [ + { + "type": "dataset", + "key": "train_data_output" + }, + { + "type": "model", + "key": "train_model_output" + }, + { + "type": "report", + "key": "metric" + } + ] + } + ], + "version": "2.0" + }, + "config": { + "initiator": { + "role": "guest", + "node_id": "JG0100001100000010" + }, + "role": { + "guest": [ + "JG0100001100000010" + ], + "host": [ + "JG0100001100000010" + ] + }, + "job_params": { + "common": { + "sync_type": "poll" + }, + "guest": { + "0": { + "resources": { + "cpu": -1, + "memory": -1, + "disk": -1 + } + } + }, + "host": { + "0": { + "resources": { + "cpu": -1, + "memory": -1, + "disk": -1 + } + } + } + }, + "task_params": { + "common": { + "psi_0":{ + }, + "sbt_0": { + } + }, + "guest": { + "0": { + "psi_0": { + "dataset_id": "test_data#guest" + } + } + }, + "host": { + "0": { + "psi_0": { + "dataset_id": "test_data#host" + } + } + } + }, + "version": "2.0.0" + } +} \ No newline at end of file diff --git "a/python/fate_flow/adapter/bfia/examples/job/fate/lr\347\256\227\346\263\225\346\217\217\350\277\260.json" "b/python/fate_flow/adapter/bfia/examples/job/fate/lr\347\256\227\346\263\225\346\217\217\350\277\260.json" new file mode 100644 index 000000000..c6c0717fc --- /dev/null +++ "b/python/fate_flow/adapter/bfia/examples/job/fate/lr\347\256\227\346\263\225\346\217\217\350\277\260.json" @@ -0,0 +1,186 @@ +{ + "componentName": "coordinated_lr", + "title": "lr", + "provider": "fate", + "version": "2.0.0", + "description": "逻辑回归算法", + "roleList": [ + "guest", + "host", + "arbiter" + ], + "desVersion": "2.0.0", + "storageEngine": [ + "s3", + "hdfs", + "eggroll" + ], + "inputParam": [ + { + "name": "learning_rate_scheduler", + "title": "", + "description": "learning rate scheduler", + "type": "string", + "optional": "true", + "defaultValue": "{'method': 'linear', 'scheduler_params': {'start_factor': 1.0}}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "epochs", + "title": "", + "description": "max iteration num", + "type": "integer", + "optional": "true", + "defaultValue": 20, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "batch_size", + "title": "", + "description": "batch size, None means full batch, otherwise should be no less than 10, default None", + "type": "integer", + "optional": "true", + "defaultValue": null, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "optimizer", + "title": "", + "description": "", + "type": "string", + "optional": "true", + "defaultValue": "{'method': 'sgd', 'penalty': 'l2', 'plpah': 1.0, 'optimizer_params': {'lr': 1e-2, 'weight_decay': 0}}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "floating_point_precision", + "title": "", + "description": "floating point precision", + "type": "integer", + "optional": "true", + "defaultValue": 23, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "tol", + "title": "", + "description": "", + "type": "float", + "optional": "true", + "defaultValue": 1e-4, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "early_stop", + "title": "", + "description": "early stopping criterion, choose from {weight_diff, diff, abs, val_metrics}", + "type": "string", + "optional": "true", + "defaultValue": "diff", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "he_param", + "title": "", + "description": "homomorphic encryption param", + "type": "string", + "optional": "true", + "defaultValue": "{'kind': 'paillier', 'key_length': 1024}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "init_param", + "title": "", + "description": "Model param init setting.", + "type": "string", + "optional": "true", + "defaultValue": "{'method': 'random_uniform', 'fit_intercept': true, 'random_state': null}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "threshold", + "title": "", + "description": "predict threshold for binary data", + "type": "float", + "optional": "true", + "defaultValue": 0.5, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + } + ], + "outputData": [ + { + "name": "train_output_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + }, + { + "name": "output_model", + "description": "模型", + "category": "model", + "dataFormat": [ + "json" + ] + }, + { + "name": "metric", + "description": "report", + "category": "report", + "dataFormat": [ + "json" + ] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] + } \ No newline at end of file diff --git "a/python/fate_flow/adapter/bfia/examples/job/fate/psi\347\256\227\346\263\225\346\217\217\350\277\260.json" "b/python/fate_flow/adapter/bfia/examples/job/fate/psi\347\256\227\346\263\225\346\217\217\350\277\260.json" new file mode 100644 index 000000000..62b047d58 --- /dev/null +++ "b/python/fate_flow/adapter/bfia/examples/job/fate/psi\347\256\227\346\263\225\346\217\217\350\277\260.json" @@ -0,0 +1,81 @@ +{ + "componentName": "psi", + "title": "对齐算法", + "provider": "fate", + "version": "2.0.0", + "description": "对齐算法", + "roleList": [ + "guest", + "host" + ], + "desVersion": "2.0.0", + "storageEngine": [ + "s3", + "hdfs", + "eggroll" + ], + "inputParam": [ + { + "name": "protocol", + "title": "protocol", + "description": "protocol", + "type": "string", + "optional": "true", + "defaultValue": "ecdh_psi", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "curve_type", + "title": "curve_type", + "description": "curve_type", + "type": "string", + "optional": "true", + "defaultValue": "curve25519", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [ + { + "name": "input_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + } + ], + "outputData": [ + { + "name": "output_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + }, + { + "name": "metric", + "description": "对齐数", + "category": "report", + "dataFormat": [ + "json" + ] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] + } \ No newline at end of file diff --git "a/python/fate_flow/adapter/bfia/examples/job/fate/sbt\347\256\227\346\263\225\346\217\217\350\277\260.json" "b/python/fate_flow/adapter/bfia/examples/job/fate/sbt\347\256\227\346\263\225\346\217\217\350\277\260.json" new file mode 100644 index 000000000..527418efa --- /dev/null +++ "b/python/fate_flow/adapter/bfia/examples/job/fate/sbt\347\256\227\346\263\225\346\217\217\350\277\260.json" @@ -0,0 +1,245 @@ +{ + "componentName": "hetero_secureboost", + "title": "sbt", + "provider": "fate", + "version": "2.0.0", + "description": "XGBoost算法", + "roleList": [ + "guest", + "host" + ], + "desVersion": "2.0.0", + "storageEngine": [ + "s3", + "hdfs", + "eggroll" + ], + "inputParam": [ + { + "name": "num_trees", + "title": "", + "description": "", + "type": "integer", + "optional": "true", + "defaultValue": 3, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "learning_rate", + "title": "", + "description": "", + "type": "float", + "optional": "true", + "defaultValue": 0.3, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "max_depth", + "title": "", + "description": "max depth of a tree", + "type": "integer", + "optional": "true", + "defaultValue": 3, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "max_bin", + "title": "", + "description": "max bin number of feature binning", + "type": "integer", + "optional": "true", + "defaultValue": 32, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "objective", + "title": "", + "description": "", + "type": "string", + "optional": "true", + "defaultValue": "binary:bce", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "num_class", + "title": "", + "description": "", + "type": "integer", + "optional": "true", + "defaultValue": 2, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "l2", + "title": "", + "description": "L2 regularization", + "type": "float", + "optional": "true", + "defaultValue": 0.1, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_impurity_split", + "title": "", + "description": "min impurity when splitting a tree node", + "type": "float", + "optional": "true", + "defaultValue": 1e-2, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_sample_split", + "title": "", + "description": "min sample to split a tree node", + "type": "integer", + "optional": "true", + "defaultValue": 2, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_leaf_node", + "title": "", + "description": "mininum sample contained in a leaf node", + "type": "integer", + "optional": "true", + "defaultValue": 1, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "min_child_weight", + "title": "", + "description": "minumum hessian contained in a leaf node", + "type": "integer", + "optional": "true", + "defaultValue": 1, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "gh_pack", + "title": "", + "description": "whether to pack gradient and hessian together", + "type": "bool", + "optional": "true", + "defaultValue": true, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "split_info_pack", + "title": "", + "description": "for host side, whether to pack split info together", + "type": "bool", + "optional": "true", + "defaultValue": true, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "hist_sub", + "title": "", + "description": "whether to use histogram subtraction", + "type": "bool", + "optional": "true", + "defaultValue": true, + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "he_param", + "title": "", + "description": "whether to use histogram subtraction", + "type": "string", + "optional": "true", + "defaultValue": "{'kind': 'paillier', 'key_length': 1024}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + } + ], + "outputData": [ + { + "name": "train_data_output", + "description": "训练集数据", + "category": "dataset", + "dataFormat": [ + "dataset" + ] + }, + { + "name": "train_model_output", + "description": "输出模型", + "category": "model", + "dataFormat": [ + "json" + ] + }, + { + "name": "metric", + "description": "对齐数", + "category": "report", + "dataFormat": [ + "json" + ] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] + } \ No newline at end of file diff --git a/python/fate_flow/adapter/bfia/examples/job/unionpay/bfia_psi_lr.json b/python/fate_flow/adapter/bfia/examples/job/unionpay/bfia_psi_lr.json new file mode 100644 index 000000000..32146e0e9 --- /dev/null +++ b/python/fate_flow/adapter/bfia/examples/job/unionpay/bfia_psi_lr.json @@ -0,0 +1,136 @@ +{ + "flow_id": "", + "dag": { + "components": [ + { + "name": "intersect_rsa_1", + "componentName": "Intersection", + "provider": "unionpay", + "version": "2.0.0", + "input": [], + "output": [ + { + "type": "dataset", + "key": "train_data" + }, + { + "type": "report", + "key": "report0" + } + ] + }, + { + "name": "hetero_logistic_regression_1", + "componentName": "HeteroLR", + "provider": "unionpay", + "version": "2.0.0", + "input": [ + { + "type": "dataset", + "key": "intersect_rsa_1.train_data" + } + ], + "output": [ + { + "type": "dataset", + "key": "data0" + }, + { + "type": "model", + "key": "model0" + }, + { + "type": "report", + "key": "report0" + } + ] + } + ], + "version": "2.0" + }, + "config": { + "initiator": { + "role": "guest", + "node_id": "JG0100001100000010" + }, + "role": { + "guest": [ + "JG0100001100000010" + ], + "host": [ + "JG0100001100000010" + ], + "arbiter": [ + "JG0100001100000010" + ] + }, + "job_params": { + "common": { + "sync_type": "poll" + }, + "guest": { + "0": { + "resources": { + "cpu": -1, + "memory": -1, + "disk": -1 + } + } + }, + "host": { + "0": { + "resources": { + "cpu": -1, + "memory": -1, + "disk": -1 + } + } + } + }, + "task_params": { + "common": { + "hetero_logistic_regression_1":{ + "id":"id", + "label":"y", + "penalty":"L2", + "tol":0.0001, + "alpha":0.01, + "optimizer":"nesterov_momentum_sgd", + "batch_size":-1, + "learning_rate":0.15, + "init_param":{ + "init_method":"zeros" + }, + "max_iter":2, + "early_stop":"diff" + }, + "intersect_rsa_1": { + "id": "id", + "intersect_method": "rsa", + "sync_intersect_ids": true, + "only_output_key": false, + "rsa_params": { + "hash_method": "sha256", + "final_hash_method": "sha256", + "key_length": 2048 + } + } + }, + "guest": { + "0": { + "intersect_rsa_1": { + "dataset_id": "testspace#test_guest" + } + } + }, + "host": { + "0": { + "intersect_rsa_1": { + "dataset_id": "testspace#test_host" + } + } + } + }, + "version": "2.0.0" + } +} \ No newline at end of file diff --git a/python/fate_flow/adapter/bfia/examples/job/unionpay/bfia_psi_sbt.json b/python/fate_flow/adapter/bfia/examples/job/unionpay/bfia_psi_sbt.json new file mode 100644 index 000000000..8b7dfb5fe --- /dev/null +++ b/python/fate_flow/adapter/bfia/examples/job/unionpay/bfia_psi_sbt.json @@ -0,0 +1,155 @@ +{ + "flow_id": "", + "dag": { + "components": [ + { + "name": "intersect_rsa_1", + "componentName": "Intersection", + "provider": "unionpay", + "version": "2.0.0", + "input": [], + "output": [ + { + "type": "dataset", + "key": "train_data" + }, + { + "type": "report", + "key": "report0" + } + ] + }, + { + "name": "hetero_secureboost_1", + "componentName": "HeteroSecureBoost", + "provider": "unionpay", + "version": "2.0.0", + "input": [ + { + "type": "dataset", + "key": "intersect_rsa_1.train_data" + } + ], + "output": [ + { + "type": "dataset", + "key": "data0" + }, + { + "type": "model", + "key": "model0" + }, + { + "type": "report", + "key": "report0" + }, + { + "type": "report", + "key": "report1" + }, + { + "type": "report", + "key": "report2" + } + ] + } + ], + "version": "2.0" + }, + "config": { + "initiator": { + "role": "guest", + "node_id": "JG0100001100000010" + }, + "role": { + "guest": [ + "JG0100001100000010" + ], + "host": [ + "JG0100001100000010" + ] + }, + "job_params": { + "common": { + "sync_type": "poll" + }, + "guest": { + "0": { + "resources": { + "cpu": -1, + "memory": -1, + "disk": -1 + } + } + }, + "host": { + "0": { + "resources": { + "cpu": -1, + "memory": -1, + "disk": -1 + } + } + } + }, + "task_params": { + "common": { + "hetero_secureboost_1": { + "id": "id", + "label": "y", + "learning_rate": 0.5, + "objective_param": { + "objective": "cross_entropy" + }, + "num_trees": 2, + "subsample_feature_rate": 1, + "n_iter_no_change": "True", + "tol": 0.0001, + "predict_param": { + "threshold": 0.5 + }, + "cv_param": { + "n_splits": 5, + "shuffle": false, + "random_seed": 103, + "need_cv": false + }, + "metrics": [ + "auc", + "ks" + ], + "early_stopping_rounds": "", + "tree_param": { + "max_depth": 5 + } + }, + "intersect_rsa_1": { + "id": "id", + "intersect_method": "rsa", + "sync_intersect_ids": true, + "only_output_key": false, + "rsa_params": { + "hash_method": "sha256", + "final_hash_method": "sha256", + "key_length": 2048 + } + } + }, + "guest": { + "0": { + "intersect_rsa_1": { + "dataset_id": "testspace#test_guest" + } + } + }, + "host": { + "0": { + "intersect_rsa_1": { + "dataset_id": "testspace#test_host" + } + } + } + }, + "version": "2.0.0" + } +} \ No newline at end of file diff --git a/python/fate_flow/adapter/bfia/examples/job/unionpay/unionpay_components.json b/python/fate_flow/adapter/bfia/examples/job/unionpay/unionpay_components.json new file mode 100644 index 000000000..30f97c03f --- /dev/null +++ b/python/fate_flow/adapter/bfia/examples/job/unionpay/unionpay_components.json @@ -0,0 +1,670 @@ +{ + "name": "unionpay", + "device": "docker", + "version": "2.0.0", + "metadata": { + "base_url": "", + "image": "unionpay:2.0.0" + }, + "protocol": "bfia", + "components_description": + { + "Intersection": { + "componentName": "Intersection", + "title": "对齐算法", + "provider": "unionpay", + "version": "2.0.0", + "description": "对齐算法", + "roleList": ["guest", "host"], + "desVersion": "1.3.0", + "storageEngine": ["s3","hdfs"], + "inputParam": [ + { + "name": "id", + "title": "id列", + "description": "id字段名", + "type": "string", + "optional": "true", + "defaultValue": "x1", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "intersect_method", + "title": "对齐方式", + "description": "对齐方式", + "type": "string", + "optional": "true", + "defaultValue": "raw", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "sync_intersect_ids", + "title": "同步对齐id", + "description": "同步对齐id", + "type": "boolean", + "optional": "true", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "only_output_key", + "title": "仅输出id", + "description": "仅输出id", + "type": "boolean", + "optional": "true", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "use_hash", + "title": "是否使用哈希", + "description": "是否使用哈希", + "type": "string", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "optional": "true", + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "hash_method", + "title": "哈希方法", + "description": "哈希方法", + "type": "string", + "optional": "true", + "defaultValue": "sha256", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "salt", + "title": "salt", + "description": "salt", + "type": "string", + "optional": "true", + "defaultValue": "12345", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "base64", + "title": "选择base64", + "description": "是否选择base64方式", + "type": "string", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "optional": "true", + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "join_role", + "title": "参与角色", + "description": "参与角色", + "type": "string", + "optional": "true", + "defaultValue": "host", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [{ + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv","yaml"] + }], + "outputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }, + { + "name": "train-intersection", + "description": "对齐数", + "category": "report", + "dataFormat": ["csv"] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] +}, + "HeteroLR": { + "componentName": "HeteroLR", + "title": "纵向逻辑回归算法", + "provider": "unionpay", + "version": "2.0.0", + "description": "纵向逻辑回归算法", + "roleList": ["guest", "host", "arbiter"], + "desVersion": "1.2.0", + "storageEngine": ["s3","hdfs"], + "inputParam": [ + { + "name": "id", + "title": "id列", + "description": "id字段名", + "type": "string", + "optional": "true", + "defaultValue": "x1", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "label", + "title": "标签", + "description": "label字段名", + "type": "string", + "optional": "true", + "defaultValue": "y", + "validator": "regular-正则项", + "dependsOn": [], + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "penalty", + "title": "正则项", + "description": "正则项", + "type": "string", + "bindingData": [ + { + "label": "L1正则", + "value": "L1" + }, + { + "label": "L2正则", + "value": "L2" + } + ], + "optional": "true", + "defaultValue": "L2", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "tol", + "title": "最小损失值", + "description": "最小损失值", + "type": "float", + "optional": "true", + "defaultValue": "0.0001", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "alpha", + "title": "惩罚因子", + "description": "惩罚因子", + "type": "float", + "optional": "true", + "defaultValue": "0.01", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "optimizer", + "title": "优化方法", + "description": "优化方法", + "type": "string", + "bindingData": [ + { + "label": "rmsprop", + "value": "rmsprop" + }, + { + "label": "sgd", + "value": "sgd" + }, + { + "label": "adam", + "value": "adam" + }, + { + "label": "sqn", + "value": "sqn" + }, + { + "label": "adagrad", + "value": "adagrad" + } + ], + "optional": "true", + "defaultValue": "rmsprop", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "batch_size", + "title": "批量梯度下降样本量", + "description": "每轮迭代抽取数据计算梯度的size", + "type": "integer", + "bindingData": [ + { + "label": "all", + "value": "all" + }, + { + "label": "2048", + "value": "2048" + }, + { + "label": "4096", + "value": "4096" + }, + { + "label": "8192", + "value": "8192" + } + ], + "optional": "true", + "defaultValue": "2048", + "validator": "(0,1000)", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "learning_rate", + "title": "学习率", + "description": "学习率", + "type": "float", + "optional": "true", + "defaultValue": "0.15", + "validator": "regular-正则项", + "dependsOn": ["optimizer.sgd", "optimizer.adam"], + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "init_param", + "title": "初始化方式", + "description": "初始化方式", + "type": "string", + "optional": "true", + "defaultValue": "zeros", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "max_iter", + "title": "迭代次数", + "description": "迭代次数", + "type": "integer", + "optional": "true", + "defaultValue": "30", + "validator": "(0,1000)", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "early_stop", + "title": "早停策略", + "description": "早停策略", + "type": "string", + "bindingData": [ + { + "label": "weight_diff", + "value": "weight_diff" + }, + { + "label": "diff", + "value": "diff" + } + ], + "optional": "true", + "defaultValue": "weight_diff", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + } + ], + "inputData": [{ + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }], + "outputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }, + { + "name": "model", + "description": "模型文件", + "category": "model", + "dataFormat": ["csv"] + }, + { + "name": "train-loss", + "description": "loss值", + "category": "metric", + "dataFormat": ["json"] + } + ], + "result": [{ + "resultCode": "4444", + "resultMessage": "算法执行失败" + }] +}, + "HeteroSecureBoost":{ + "componentName": "HeteroSecureBoost", + "title": "XGBoost算法", + "provider": "unionpay", + "version": "2.0.0", + "description": "XGBoost算法", + "roleList": ["guest", "host"], + "desVersion": "1.3.0", + "storageEngine": ["s3","hdfs"], + "inputParam": [ + { + "name": "objective_param", + "title": "目标参数", + "description": "目标参数", + "type": "string", + "optional": "true", + "bindingData": [ + { + "label": "cross_entropy", + "value": "cross_entropy" + } + ], + "defaultValue": "cross_entropy", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "learning_rate", + "title": "学习率", + "description": "学习率", + "type": "float", + "optional": "true", + "defaultValue": "0.15", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "num_trees", + "title": "树个数", + "description": "树个数", + "type": "integer", + "optional": "true", + "defaultValue": "5", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "subsample_feature_rate", + "title": "子样本率", + "description": "子样本率", + "type": "integer", + "optional": "true", + "defaultValue": "1", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "n_iter_no_change", + "title": "n轮无变化", + "description": "n轮无变化", + "type": "boolean", + "optional": "true", + "bindingData": [ + { + "label": "是", + "value": "true" + }, + { + "label": "否", + "value": "false" + } + ], + "defaultValue": "true", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "tol", + "title": "停止容忍度", + "description": "停止容忍度", + "type": "float", + "optional": "true", + "defaultValue": "0.0001", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "bin_num", + "title": "分位数", + "description": "分位数", + "type": "integer", + "optional": "true", + "defaultValue": "32", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "predict_param", + "title": "预测参数", + "description": "预测参数", + "type": "string", + "optional": "true", + "defaultValue": "{\"threshold\": 0.5}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "cv_param", + "title": "cv参数", + "description": "cv参数", + "type": "string", + "optional": "true", + "defaultValue": "{\"n_splits\": 5, \"shuffle\": false, \"random_seed\": 103, \"need_cv\": false}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "metrics", + "title": "计算指标", + "description": "计算指标", + "type": "string", + "optional": "true", + "defaultValue": "[\"auc\", \"ks\"]", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + }, + { + "name": "early_stop", + "title": "早停策略", + "description": "早停策略", + "type": "string", + "optional": "true", + "bindingData": [ + { + "label": "weight_diff", + "value": "weight_diff" + }, + { + "label": "diff", + "value": "diff" + } + ], + "defaultValue": "weight_diff", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "select" + }, + { + "name": "tree_param", + "title": "树参数", + "description": "树参数", + "type": "string", + "optional": "true", + "defaultValue": "{\"max_depth\": 3}", + "validator": "regular-正则项", + "UIPattern": "editeable", + "groupTag": "默认分组-显示", + "UIType": "input" + } + ], + "inputData": [{ + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }], + "outputData": [ + { + "name": "train_data", + "description": "训练集数据", + "category": "dataset", + "dataFormat": ["csv"] + }, + { + "name": "model", + "description": "模型文件", + "category": "model", + "dataFormat": ["csv"] + }, + { + "name": "train-loss", + "description": "loss值", + "category": "report", + "dataFormat": ["json"] + }, + { + "name": "train-auc", + "description": "auc ks值", + "category": "report", + "dataFormat": ["json"] + }, + { + "name": "train-ks", + "description": "ks曲线值", + "category": "report", + "dataFormat": ["json"] + } + ], + "result": [ + { + "resultCode": "00000000", + "resultMessage": "成功" + }, + { + "resultCode": "99999999", + "resultMessage": "算法执行失败" + } + ] +} + } + +} \ No newline at end of file diff --git a/python/fate_flow/adapter/bfia/examples/test/data/guest/data_0 b/python/fate_flow/adapter/bfia/examples/test/data/guest/data_0 new file mode 100644 index 000000000..eb483aadc Binary files /dev/null and b/python/fate_flow/adapter/bfia/examples/test/data/guest/data_0 differ diff --git a/python/fate_flow/adapter/bfia/examples/test/data/guest/metadata b/python/fate_flow/adapter/bfia/examples/test/data/guest/metadata new file mode 100644 index 000000000..be76fad45 --- /dev/null +++ b/python/fate_flow/adapter/bfia/examples/test/data/guest/metadata @@ -0,0 +1 @@ +{"column_info": [], "options": "", "partitions": 1, "description": "", "data_type": "dense", "file_type": "", "create_time": 1699931100.544714, "expire_time": 1731467100.544714, "count": 0, "metadata": {"data_overview": null, "index": null, "metadata": {"options": {"partitions": 16}, "schema": {"partition_order_mappings": {"ef2607a3ee835b1f6714fa9fa4a85ee40": {"end_block_id": 0, "end_index": 37, "start_block_id": 0, "start_index": 0}, "ef2607a3ee835b1f6714fa9fa4a85ee41": {"end_block_id": 1, "end_index": 69, "start_block_id": 1, "start_index": 38}, "ef2607a3ee835b1f6714fa9fa4a85ee410": {"end_block_id": 2, "end_index": 104, "start_block_id": 2, "start_index": 70}, "ef2607a3ee835b1f6714fa9fa4a85ee412": {"end_block_id": 3, "end_index": 137, "start_block_id": 3, "start_index": 105}, "ef2607a3ee835b1f6714fa9fa4a85ee416": {"end_block_id": 4, "end_index": 181, "start_block_id": 4, "start_index": 138}, "ef2607a3ee835b1f6714fa9fa4a85ee42": {"end_block_id": 5, "end_index": 215, "start_block_id": 5, "start_index": 182}, "ef2607a3ee835b1f6714fa9fa4a85ee420": {"end_block_id": 6, "end_index": 253, "start_block_id": 6, "start_index": 216}, "ef2607a3ee835b1f6714fa9fa4a85ee43": {"end_block_id": 7, "end_index": 290, "start_block_id": 7, "start_index": 254}, "ef2607a3ee835b1f6714fa9fa4a85ee435": {"end_block_id": 8, "end_index": 323, "start_block_id": 8, "start_index": 291}, "ef2607a3ee835b1f6714fa9fa4a85ee44": {"end_block_id": 9, "end_index": 365, "start_block_id": 9, "start_index": 324}, "ef2607a3ee835b1f6714fa9fa4a85ee441": {"end_block_id": 10, "end_index": 407, "start_block_id": 10, "start_index": 366}, "ef2607a3ee835b1f6714fa9fa4a85ee46": {"end_block_id": 11, "end_index": 432, "start_block_id": 11, "start_index": 408}, "ef2607a3ee835b1f6714fa9fa4a85ee465": {"end_block_id": 12, "end_index": 465, "start_block_id": 12, "start_index": 433}, "ef2607a3ee835b1f6714fa9fa4a85ee47": {"end_block_id": 13, "end_index": 497, "start_block_id": 13, "start_index": 466}, "ef2607a3ee835b1f6714fa9fa4a85ee48": {"end_block_id": 14, "end_index": 535, "start_block_id": 14, "start_index": 498}, "ef2607a3ee835b1f6714fa9fa4a85ee49": {"end_block_id": 15, "end_index": 568, "start_block_id": 15, "start_index": 536}}, "schema_meta": {"anonymous_summary": {"column_len": 10, "site_name": null}, "fields": [{"dtype": "index", "name": "extend_sid", "property": "sample_id", "should_compress": false}, {"dtype": "index", "name": "id", "property": "match_id", "should_compress": false}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_y", "dtype": "int32", "name": "y", "property": "label", "should_compress": false}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x0", "dtype": "float32", "name": "x0", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x1", "dtype": "float32", "name": "x1", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x2", "dtype": "float32", "name": "x2", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x3", "dtype": "float32", "name": "x3", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x4", "dtype": "float32", "name": "x4", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x5", "dtype": "float32", "name": "x5", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x6", "dtype": "float32", "name": "x6", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x7", "dtype": "float32", "name": "x7", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x8", "dtype": "float32", "name": "x8", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x9", "dtype": "float32", "name": "x9", "property": "column", "should_compress": true}]}, "type": "fate.arch.dataframe"}}, "model_key": null, "model_overview": {}, "name": null, "namespace": null, "source": {"component": "dataframe_transformer", "output_artifact_key": "dataframe_output", "output_index": null, "party_task_id": "202311011508048482780_transformer_0_0_local_0", "task_id": "202311011508048482780_transformer_0", "task_name": "transformer_0"}, "type_name": null}} \ No newline at end of file diff --git a/python/fate_flow/adapter/bfia/examples/test/data/host/data_0 b/python/fate_flow/adapter/bfia/examples/test/data/host/data_0 new file mode 100644 index 000000000..61519f471 Binary files /dev/null and b/python/fate_flow/adapter/bfia/examples/test/data/host/data_0 differ diff --git a/python/fate_flow/adapter/bfia/examples/test/data/host/metadata b/python/fate_flow/adapter/bfia/examples/test/data/host/metadata new file mode 100644 index 000000000..cbb3542e2 --- /dev/null +++ b/python/fate_flow/adapter/bfia/examples/test/data/host/metadata @@ -0,0 +1 @@ +{"column_info": [], "options": "", "partitions": 1, "description": "", "data_type": "dense", "file_type": "", "create_time": 1699931357.999101, "expire_time": 1731467357.999101, "count": 0, "metadata": {"data_overview": null, "index": null, "metadata": {"options": {"partitions": 16}, "schema": {"partition_order_mappings": {"2e8078183e0e20aa9ed1fd20929a4e180": {"end_block_id": 0, "end_index": 29, "start_block_id": 0, "start_index": 0}, "2e8078183e0e20aa9ed1fd20929a4e181": {"end_block_id": 1, "end_index": 75, "start_block_id": 1, "start_index": 30}, "2e8078183e0e20aa9ed1fd20929a4e1811": {"end_block_id": 2, "end_index": 104, "start_block_id": 2, "start_index": 76}, "2e8078183e0e20aa9ed1fd20929a4e1812": {"end_block_id": 3, "end_index": 150, "start_block_id": 3, "start_index": 105}, "2e8078183e0e20aa9ed1fd20929a4e1815": {"end_block_id": 4, "end_index": 189, "start_block_id": 4, "start_index": 151}, "2e8078183e0e20aa9ed1fd20929a4e1817": {"end_block_id": 5, "end_index": 223, "start_block_id": 5, "start_index": 190}, "2e8078183e0e20aa9ed1fd20929a4e182": {"end_block_id": 6, "end_index": 267, "start_block_id": 6, "start_index": 224}, "2e8078183e0e20aa9ed1fd20929a4e1823": {"end_block_id": 7, "end_index": 298, "start_block_id": 7, "start_index": 268}, "2e8078183e0e20aa9ed1fd20929a4e1825": {"end_block_id": 8, "end_index": 323, "start_block_id": 8, "start_index": 299}, "2e8078183e0e20aa9ed1fd20929a4e183": {"end_block_id": 9, "end_index": 363, "start_block_id": 9, "start_index": 324}, "2e8078183e0e20aa9ed1fd20929a4e1837": {"end_block_id": 10, "end_index": 395, "start_block_id": 10, "start_index": 364}, "2e8078183e0e20aa9ed1fd20929a4e1838": {"end_block_id": 11, "end_index": 421, "start_block_id": 11, "start_index": 396}, "2e8078183e0e20aa9ed1fd20929a4e186": {"end_block_id": 12, "end_index": 454, "start_block_id": 12, "start_index": 422}, "2e8078183e0e20aa9ed1fd20929a4e188": {"end_block_id": 13, "end_index": 497, "start_block_id": 13, "start_index": 455}, "2e8078183e0e20aa9ed1fd20929a4e1885": {"end_block_id": 14, "end_index": 529, "start_block_id": 14, "start_index": 498}, "2e8078183e0e20aa9ed1fd20929a4e189": {"end_block_id": 15, "end_index": 568, "start_block_id": 15, "start_index": 530}}, "schema_meta": {"anonymous_summary": {"column_len": 20, "site_name": null}, "fields": [{"dtype": "index", "name": "extend_sid", "property": "sample_id", "should_compress": false}, {"dtype": "index", "name": "id", "property": "match_id", "should_compress": false}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x0", "dtype": "float32", "name": "x0", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x1", "dtype": "float32", "name": "x1", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x2", "dtype": "float32", "name": "x2", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x3", "dtype": "float32", "name": "x3", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x4", "dtype": "float32", "name": "x4", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x5", "dtype": "float32", "name": "x5", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x6", "dtype": "float32", "name": "x6", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x7", "dtype": "float32", "name": "x7", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x8", "dtype": "float32", "name": "x8", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x9", "dtype": "float32", "name": "x9", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x10", "dtype": "float32", "name": "x10", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x11", "dtype": "float32", "name": "x11", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x12", "dtype": "float32", "name": "x12", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x13", "dtype": "float32", "name": "x13", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x14", "dtype": "float32", "name": "x14", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x15", "dtype": "float32", "name": "x15", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x16", "dtype": "float32", "name": "x16", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x17", "dtype": "float32", "name": "x17", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x18", "dtype": "float32", "name": "x18", "property": "column", "should_compress": true}, {"anonymous_name": "AnonymousRole_AnonymousPartyId_x19", "dtype": "float32", "name": "x19", "property": "column", "should_compress": true}]}, "type": "fate.arch.dataframe"}}, "model_key": null, "model_overview": {}, "name": null, "namespace": null, "source": {"component": "dataframe_transformer", "output_artifact_key": "dataframe_output", "output_index": null, "party_task_id": "202311011508108546800_transformer_0_0_local_0", "task_id": "202311011508108546800_transformer_0", "task_name": "transformer_0"}, "type_name": null}} \ No newline at end of file diff --git a/python/fate_flow/adapter/bfia/examples/test/test_job.py b/python/fate_flow/adapter/bfia/examples/test/test_job.py new file mode 100644 index 000000000..efa850c4f --- /dev/null +++ b/python/fate_flow/adapter/bfia/examples/test/test_job.py @@ -0,0 +1,39 @@ +import json + +import requests + +base_url = "http://127.0.0.1:9380" + + +def register_components(): + uri = "/v2/provider/register" + config_path = "../job/unionpay/unionpay_components.json" + body = json.load(open(config_path, "r")) + resp = requests.post(base_url+uri, json=body) + print(resp.text) + + +register_components() + + +def submit_job(): + uri = "/v1/platform/schedule/job/create_all" + config_path = "../job/unionpay/bfia_psi_sbt.json" + body = json.load(open(config_path, "r")) + resp = requests.post(base_url+uri, json=body) + print(resp.text) + + +def start_job(job_id): + uri = "/v1/interconn/schedule/job/start" + resp = requests.post(base_url+uri, json={"job_id": job_id}) + print(resp.text) + + +def stop_job(job_id): + uri = "/v1/interconn/schedule/job/stop_all" + resp = requests.post(base_url+uri, json={"job_id": job_id}) + print(resp.text) + + +submit_job() diff --git a/python/fate_flow/adapter/bfia/examples/test/test_task.py b/python/fate_flow/adapter/bfia/examples/test/test_task.py new file mode 100644 index 000000000..465a57315 --- /dev/null +++ b/python/fate_flow/adapter/bfia/examples/test/test_task.py @@ -0,0 +1,22 @@ +import json + +import requests + +base_url = "http://127.0.0.1:9380" + + +def start_task(job_id, task_id, task_name): + uri = "/v1/interconn/schedule/task/start" + resp = requests.post(base_url+uri, json={"job_id": job_id, "task_id": task_id, "task_name": task_name}) + print(resp.text) + + +def stop_job(job_id): + uri = "/v1/platform/schedule/job/stop_all" + resp = requests.post(base_url+uri, json={"job_id": job_id}) + print(resp.text) + + +# submit_job() +# start_task("202310161542273925260", "202310161422165657200_wzh3", "intersect_rsa_1") +# stop_job("202310161542273925260") diff --git a/python/fate_flow/adapter/bfia/examples/test/upload/upload_guest.py b/python/fate_flow/adapter/bfia/examples/test/upload/upload_guest.py new file mode 100644 index 000000000..fb5b5cb27 --- /dev/null +++ b/python/fate_flow/adapter/bfia/examples/test/upload/upload_guest.py @@ -0,0 +1,18 @@ +import yaml + +from fate_flow.adapter.bfia import engine_storage + +url = "http://127.0.0.1:9000" +username = "admin" +password = "12345678" + +session = engine_storage.session.S3Session(url=url, username=username, password=password) + +namespace = "test_data" +name = "guest" +fp = open("../data/guest/metadata") +metadata = yaml.safe_load(fp) +print(metadata) + +table = session.create_table(name=name, namespace=namespace, metadata=metadata.get("metadata"), column_info=[]) +table.upload("../data/guest/data_0") diff --git a/python/fate_flow/adapter/bfia/examples/test/upload/upload_host.py b/python/fate_flow/adapter/bfia/examples/test/upload/upload_host.py new file mode 100644 index 000000000..285953baa --- /dev/null +++ b/python/fate_flow/adapter/bfia/examples/test/upload/upload_host.py @@ -0,0 +1,18 @@ +import yaml + +from fate_flow.adapter.bfia import engine_storage + +url = "http://127.0.0.1:9000" +username = "admin" +password = "12345678" + +session = engine_storage.session.S3Session(url=url, username=username, password=password) + +namespace = "test_data" +name = "host" +fp = open("../data/host/metadata") +metadata = yaml.safe_load(fp) +print(metadata) + +table = session.create_table(name=name, namespace=namespace, column_info=[], metadata=metadata.get("metadata")) +table.upload("../data/host/data_0") diff --git a/python/fate_flow/adapter/bfia/runtime_config.py b/python/fate_flow/adapter/bfia/runtime_config.py new file mode 100644 index 000000000..688aa81c8 --- /dev/null +++ b/python/fate_flow/adapter/bfia/runtime_config.py @@ -0,0 +1,38 @@ +import os.path + +from fate_flow.runtime.system_settings import HOST, HTTP_PORT +from fate_flow.adapter.bfia.settings import LOCAL_SITE_ID as PARTY_ID +from fate_flow.settings import HTTP_REQUEST_TIMEOUT +from fate_flow.utils.file_utils import load_yaml_conf +from ofx.api.client import BfiaSchedulerApi + + +class BfiaRuntimeConfig(object): + ROUTE_TABLE: dict = {} + SCHEDULE_CLIENT: BfiaSchedulerApi = None + + @classmethod + def load_route_table_from_file(cls): + name = "route_table.yaml" + route_table_path = os.path.join(os.path.dirname(__file__), "conf", name) + cls.ROUTE_TABLE.update(load_yaml_conf(route_table_path)) + + @classmethod + def set_schedule_client(cls, schedule_client): + cls.SCHEDULE_CLIENT = schedule_client + + @classmethod + def init(cls): + # init route table + cls.load_route_table_from_file() + + # init schedule client + cls.set_schedule_client( + BfiaSchedulerApi( + host=HOST, + port=HTTP_PORT, + timeout=HTTP_REQUEST_TIMEOUT, + route_table=cls.ROUTE_TABLE, + self_node_id=PARTY_ID + ) + ) diff --git a/python/fate_flow/adapter/bfia/scheduler/__init__.py b/python/fate_flow/adapter/bfia/scheduler/__init__.py new file mode 100644 index 000000000..4562c3508 --- /dev/null +++ b/python/fate_flow/adapter/bfia/scheduler/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .detector import BfiaDetector +from .scheduler import BfiaScheduler + +__all__ = [BfiaScheduler, BfiaDetector] diff --git a/python/fate_flow/adapter/bfia/scheduler/detector.py b/python/fate_flow/adapter/bfia/scheduler/detector.py new file mode 100644 index 000000000..d6b9d7a99 --- /dev/null +++ b/python/fate_flow/adapter/bfia/scheduler/detector.py @@ -0,0 +1,71 @@ +import time + +from fate_flow.adapter.bfia.utils.entity.status import TaskStatus +from fate_flow.adapter.bfia.wheels.federated import BfiaFederatedScheduler +from fate_flow.adapter.bfia.wheels.saver import BfiaJobSaver +from fate_flow.engine.devices import build_engine +from fate_flow.utils.cron import Cron +from fate_flow.utils.log_utils import detect_logger + + +class BfiaDetector(Cron): + def run_do(self): + self.logger = detect_logger(log_type="bfia_detector") + self.detect_running_task() + + def detect_running_task(self): + self.logger.info('start to detect running task..') + count = 0 + try: + running_tasks = BfiaJobSaver.query_task(party_status=TaskStatus.RUNNING) + self.logger.info(f'running task: {running_tasks}') + stop_job_ids = set() + for task in running_tasks: + try: + process_exist = build_engine(task.f_provider_name).is_alive(task) + if not process_exist: + msg = f"task {task.f_task_id} {task.f_task_version} on {task.f_role} {task.f_party_id}" + detect_logger(job_id=task.f_job_id).info( + f"{msg} with {task.f_party_status} process {task.f_run_pid} does not exist") + time.sleep(3) + _tasks = BfiaJobSaver.query_task(task_id=task.f_task_id, task_version=task.f_task_version, + role=task.f_role, party_id=task.f_party_id) + if _tasks: + if _tasks[0].f_party_status == TaskStatus.RUNNING: + stop_job_ids.add(task.f_job_id) + detect_logger(job_id=task.f_job_id).info( + f"{msg} party status has been checked twice, try to stop job") + else: + detect_logger(job_id=task.f_job_id).info( + f"{msg} party status has changed to {_tasks[0].f_party_status}, may be stopped by task_controller.stop_task, pass stop job again") + else: + detect_logger(job_id=task.f_job_id).warning(f"{msg} can not found on db") + except Exception as e: + detect_logger(job_id=task.f_job_id).exception(e) + if stop_job_ids: + self.logger.info('start to stop jobs: {}'.format(stop_job_ids)) + stop_jobs = set() + for job_id in stop_job_ids: + jobs = BfiaJobSaver.query_job(job_id=job_id) + if jobs: + stop_jobs.add(jobs[0]) + self.request_stop_jobs(jobs=stop_jobs, stop_msg="task executor process abort") + except Exception as e: + self.logger.exception(e) + finally: + self.logger.info(f"finish detect {count} running task") + + def request_stop_jobs(self, jobs, stop_msg): + if not len(jobs): + return + self.logger.info(f"have {len(jobs)} should be stopped, because of {stop_msg}") + for job in jobs: + try: + detect_logger(job_id=job.f_job_id).info( + f"detector request start to stop job {job.f_job_id}, because of {stop_msg}") + status = BfiaFederatedScheduler.request_stop_job(party_id=job.f_scheduler_party_id, job_id=job.f_job_id) + detect_logger(job_id=job.f_job_id).info(f"detector request stop job {job.f_job_id} {status}") + except Exception as e: + detect_logger(job_id=job.f_job_id).exception(e) + + diff --git a/python/fate_flow/adapter/bfia/scheduler/scheduler.py b/python/fate_flow/adapter/bfia/scheduler/scheduler.py new file mode 100644 index 000000000..612627210 --- /dev/null +++ b/python/fate_flow/adapter/bfia/scheduler/scheduler.py @@ -0,0 +1,473 @@ +from fate_flow.adapter.bfia.utils.entity.code import ReturnCode +from fate_flow.adapter.bfia.utils.entity.status import StatusSet, TaskStatus, JobStatus, InterruptStatus, EndStatus +from fate_flow.adapter.bfia.wheels.federated import BfiaFederatedScheduler +from fate_flow.adapter.bfia.wheels.job import BfiaJobController +from fate_flow.adapter.bfia.utils.spec.job import DagSchemaSpec +from fate_flow.adapter.bfia.wheels.parser import get_dag_parser, translate_bfia_dag_to_dag +from fate_flow.adapter.bfia.wheels.saver import BfiaScheduleJobSaver +from fate_flow.adapter.bfia.wheels.task import BfiaTaskController +from fate_flow.db import ScheduleJob, ScheduleTask, ScheduleTaskStatus +from fate_flow.entity.code import SchedulingStatusCode +from fate_flow.entity.types import FederatedCommunicationType +from fate_flow.scheduler import SchedulerABC +from fate_flow.utils.log_utils import schedule_logger, exception_to_trace_string + + +class BfiaScheduler(SchedulerABC): + def run_do(self): + logger = schedule_logger(name="bfia_scheduler") + logger.info("start schedule bfia job") + jobs = BfiaScheduleJobSaver.query_job( + status=JobStatus.READY, + order_by=["priority", "create_time"], + reverse=[True, False] + ) + logger.info(f"have {len(jobs)} ready jobs") + if len(jobs): + job = jobs[0] + logger.info(f"schedule ready job {job.f_job_id}") + try: + self.schedule_ready_jobs(job) + except Exception as e: + schedule_logger(job.f_job_id).exception(e) + schedule_logger().info("schedule ready jobs finished") + + # running + schedule_logger().info("start schedule running jobs") + jobs = BfiaScheduleJobSaver.query_job(status=JobStatus.RUNNING, order_by="create_time", reverse=False) + schedule_logger().info(f"have {len(jobs)} running jobs") + for job in jobs: + schedule_logger().info(f"schedule running job {job.f_job_id}") + try: + self.schedule_running_job(job=job) + except Exception as e: + schedule_logger(job.f_job_id).exception(e) + schedule_logger(job.f_job_id).error("schedule job failed") + schedule_logger().info("schedule running jobs finished") + + @classmethod + def schedule_ready_jobs(cls, job: ScheduleJob): + job_id = job.f_job_id + schedule_logger(job_id).info(f"start job {job_id}") + response = BfiaFederatedScheduler.start_job(job_id=job_id, node_list=job.f_parties) + schedule_logger(job_id).info(f"start job {job_id} response: {response}") + BfiaScheduleJobSaver.update_job_status(job_info={"job_id": job_id, "status": StatusSet.RUNNING}) + + def schedule_running_job(self, job: ScheduleJob): + schedule_logger(job.f_job_id).info("scheduling running job") + task_scheduling_status_code, tasks = BfiaTaskScheduler.schedule(job=job) + tasks_status = dict([(task.f_task_name, task.f_status) for task in tasks]) + schedule_logger(job_id=job.f_job_id).info(f"task_scheduling_status_code: {task_scheduling_status_code}, " + f"tasks_status: {tasks_status.values()}") + + new_job_status = self.calculate_job_status( + task_scheduling_status_code=task_scheduling_status_code, + tasks_status=tasks_status.values() + ) + + total, finished_count = self.calculate_job_progress(tasks_status=tasks_status) + new_progress = float(finished_count) / total * 100 + schedule_logger(job.f_job_id).info( + f"job status is {new_job_status}, calculate by task status list: {tasks_status}") + if new_job_status != job.f_status or new_progress != job.f_progress: + if int(new_progress) - job.f_progress > 0: + job.f_progress = new_progress + self.update_job_on_scheduler(schedule_job=job, update_fields=["progress"]) + if new_job_status != job.f_status: + job.f_status = new_job_status + self.update_job_on_scheduler(schedule_job=job, update_fields=["status"]) + if EndStatus.contains(job.f_status): + self.finish(job=job) + schedule_logger(job.f_job_id).info("finish scheduling running job") + + @classmethod + def create_all_job(cls, dag, job_id=None): + dag_schema = DagSchemaSpec(**dag) + schedule_logger(job_id).info(f"[scheduler]start create all job, dag {dag}") + submit_result = { + "job_id": job_id, + "data": {} + } + try: + job = ScheduleJob() + job.f_job_id = job_id + job.f_parties = BfiaJobController.get_job_parties(dag_schema) + job.f_initiator_party_id = dag_schema.dag.config.initiator.node_id + job.f_scheduler_party_id = dag_schema.dag.config.initiator.node_id + job.f_dag = dag_schema.dict() + job.f_protocol = dag_schema.kind + job.f_status = StatusSet.READY + BfiaScheduleJobSaver.create_job(job.to_human_model_dict()) + body = dag_schema.dag.dict(exclude_unset=True) + body.update({ + "job_id": job_id + }) + response = BfiaFederatedScheduler.create_job( + node_list=job.f_parties, command_body=body + ) + for node_id, resp in response.items(): + if resp.get("code") != ReturnCode.SUCCESS: + # stop + raise RuntimeError(response) + + job.f_status = StatusSet.READY + cls.create_schedule_tasks(job, dag_schema) + schedule_logger(job_id).info(f"[scheduler]submit job successfully, job id is {job.f_job_id}") + result = { + "code": ReturnCode.SUCCESS, + } + submit_result.update(result) + except Exception as e: + schedule_logger(job_id).exception(e) + submit_result["code"] = ReturnCode.FAILED + submit_result["msg"] = exception_to_trace_string(e) + return submit_result + + @classmethod + def stop_all_job(cls, job_id, task_name=None): + jobs = BfiaScheduleJobSaver.query_job(job_id=job_id) + schedule_logger(job_id).info(f"[scheduler]start to stop all job") + if jobs: + job = jobs[0] + body = { + "job_id": job_id + } + if task_name: + body.update({ + "task_name": task_name + }) + resp = BfiaFederatedScheduler.stop_job( + node_list=job.f_parties, command_body=body + ) + schedule_logger(job_id).info(f"[scheduler]stop job response: {resp}") + + # update scheduler status + BfiaScheduleJobSaver.update_job_status(dict( + job_id=job_id, + status=JobStatus.FINISHED + )) + + task_info = { + "job_id": job_id + } + if task_name: + task_info.update(dict(task_name=task_name)) + else: + task_info.update(dict(status=TaskStatus.RUNNING)) + tasks = BfiaScheduleJobSaver.query_task(**task_info) + + for task in tasks: + BfiaScheduleJobSaver.update_task_status( + task_info=dict( + role=task.f_role, + party_id=task.f_party_id, + job_id=job_id, + task_id=task.f_task_id, + task_version=task.f_task_version, + status=TaskStatus.FAILED) + ) + + BfiaScheduleJobSaver.update_task_status( + task_info=dict( + job_id=job_id, + task_id=task.f_task_id, + task_version=task.f_task_version, + status=TaskStatus.FAILED), + scheduler_status=True + ) + + else: + schedule_logger(job_id).exception(f"[scheduler]No found job {job_id}") + + @classmethod + def query_job_status(cls, job_id): + jobs = BfiaScheduleJobSaver.query_job(job_id=job_id) + if jobs: + job = jobs[0] + job_status = job.f_status + all_task_status = {} + tasks = BfiaScheduleJobSaver.query_task(scheduler_status=True, job_id=job_id) + for task in tasks: + all_task_status[task.f_task_name] = task.f_status + return { + "job_status": job_status, + "status": all_task_status + } + return {} + + @classmethod + def audit_confirm(cls, job_id, status): + return BfiaScheduleJobSaver.update_job_status(job_info={"job_id": job_id, "status": status}) + + @classmethod + def callback_task(cls, task_id, role, status, node_id): + task = BfiaScheduleJobSaver.query_task(task_id=task_id, party_id=node_id)[0] + status = BfiaScheduleJobSaver.update_task_status(task_info={ + "job_id": task.f_job_id, + "role": "", + "party_id": node_id, + "task_id": task_id, + "task_version": 0, + "status": status + }) + return status + + @classmethod + def create_schedule_tasks(cls, job, dag_schema: DagSchemaSpec): + job_parser = get_dag_parser(dag_schema) + task_list = [task for task in job_parser.topological_sort()] + task_parties = {} + # get task parties + for name in task_list: + task_parties[name] = [] + for party in job.f_parties: + if job_parser.get_runtime_roles_on_party(task_name=name, party_id=party): + task_parties[name].append(party) + + # create schedule task + task_ids = {} + for name, parties in task_parties.items(): + task_id = BfiaTaskController.generate_task_id() + task_ids[name] = task_id + for node_id in parties: + cls.create_task( + job.f_job_id, + task_id, + node_id, + name, + job_parser, + parties=parties + ) + cls.create_scheduler_tasks_status(job.f_job_id, task_list, dag_schema, task_ids) + + @classmethod + def create_task(cls, job_id, task_id, node_id, task_name, job_parser, parties, task_version=0): + task = ScheduleTask() + task.f_job_id = job_id + task.f_role = "" + task.f_party_id = node_id + task.f_task_name = task_name + task.f_component = job_parser.get_component_ref(task_name=task_name) + task.f_task_id = task_id + task.f_task_version = task_version + task.f_status = TaskStatus.READY + task.f_parties = parties + BfiaScheduleJobSaver.create_task(task.to_human_model_dict()) + + @classmethod + def create_scheduler_tasks_status(cls, job_id, task_list, dag_schema: DagSchemaSpec, task_ids, + task_version=0, auto_retries=0, task_name=None): + schedule_logger(job_id).info("start create schedule task status info") + if task_name: + task_list = [task_name] + for _task_name in task_list: + task = ScheduleTaskStatus() + task.f_job_id = job_id + task.f_task_name = _task_name + task.f_task_id = task_ids.get(_task_name) + task.f_task_version = task_version + task.f_status = TaskStatus.READY + task.f_auto_retries = auto_retries + task.f_sync_type = dag_schema.dag.config.job_params.common.sync_type + status = BfiaScheduleJobSaver.create_task_scheduler_status(task.to_human_model_dict()) + schedule_logger(job_id).info(f"create schedule task {_task_name} status {status}") + + schedule_logger(job_id).info(f"create schedule task status success： {task_list}") + + @classmethod + def calculate_job_status(cls, task_scheduling_status_code, tasks_status): + tmp_status_set = set(tasks_status) + if len(tmp_status_set) == 1: + return tmp_status_set.pop() + else: + if TaskStatus.RUNNING in tmp_status_set: + return JobStatus.RUNNING + if TaskStatus.PENDING in tmp_status_set or TaskStatus.READY in tmp_status_set: + if task_scheduling_status_code == SchedulingStatusCode.HAVE_NEXT: + return JobStatus.RUNNING + for status in sorted(InterruptStatus.status_list(), key=lambda s: StatusSet.get_level(status=s), + reverse=True): + if status in tmp_status_set: + return status + raise Exception("calculate job status failed, all task status: {}".format(tasks_status)) + + @classmethod + def calculate_job_progress(cls, tasks_status): + total = 0 + finished_count = 0 + for task_status in tasks_status.values(): + total += 1 + if EndStatus.contains(task_status): + finished_count += 1 + return total, finished_count + + @classmethod + def update_job_on_scheduler(cls, schedule_job: ScheduleJob, update_fields: list): + schedule_logger(schedule_job.f_job_id).info(f"try to update job {update_fields} on scheduler") + jobs = BfiaScheduleJobSaver.query_job(job_id=schedule_job.f_job_id) + if not jobs: + raise Exception("Failed to update job status on scheduler") + job_info = schedule_job.to_human_model_dict(only_primary_with=update_fields) + for field in update_fields: + job_info[field] = getattr(schedule_job, "f_%s" % field) + if "status" in update_fields: + BfiaScheduleJobSaver.update_job_status(job_info=job_info) + BfiaScheduleJobSaver.update_job(job_info=job_info) + schedule_logger(schedule_job.f_job_id).info(f"update job {update_fields} on scheduler finished") + + @classmethod + def finish(cls, job): + schedule_logger(job.f_job_id).info(f"job finished, do something...") + cls.stop_all_job(job_id=job.f_job_id) + schedule_logger(job.f_job_id).info(f"done") + + +class BfiaTaskScheduler(object): + @classmethod + def schedule(cls, job): + schedule_logger(job.f_job_id).info("scheduling job tasks") + dag_schema = DagSchemaSpec(**job.f_dag) + job_parser = get_dag_parser(dag_schema) + tasks_group = BfiaScheduleJobSaver.get_status_tasks_asc(job_id=job.f_job_id) + schedule_logger(job.f_job_id).info(f"tasks group: {tasks_group}") + waiting_tasks = {} + job_interrupt = False + for task in tasks_group.values(): + if task.f_sync_type == FederatedCommunicationType.POLL: + if task.f_status in [TaskStatus.RUNNING]: + cls.collect_task_of_all_party(task=task, parties=job.f_parties) + else: + pass + new_task_status = cls.get_federated_task_status( + task_name=task.f_task_name, + job_id=task.f_job_id, + task_id=task.f_task_id, + task_version=task.f_task_version + ) + task_interrupt = False + task_status_have_update = False + if new_task_status != task.f_status: + task_status_have_update = True + schedule_logger(job.f_job_id).info(f"sync task status {task.f_status} to {new_task_status}") + task.f_status = new_task_status + BfiaScheduleJobSaver.update_task_status(task.to_human_model_dict(), scheduler_status=True) + if InterruptStatus.contains(new_task_status): + task_interrupt = True + job_interrupt = True + if task.f_status == TaskStatus.READY: + waiting_tasks[task.f_task_name] = task + elif task_status_have_update and EndStatus.contains(task.f_status) or task_interrupt: + schedule_logger(task.f_job_id).info(f"stop task with status: {task.f_status}") + BfiaFederatedScheduler.stop_job( + node_list=job.f_parties, + command_body={ + "job_id": task.f_job_id, + "task_name": task.f_task_name + }) + scheduling_status_code = SchedulingStatusCode.NO_NEXT + schedule_logger(job.f_job_id).info(f"job interrupt status {job_interrupt}") + schedule_logger(job.f_job_id).info(f"waiting tasks: {waiting_tasks}") + if not job_interrupt: + for task_id, waiting_task in waiting_tasks.items(): + + dependent_tasks = job_parser.infer_dependent_tasks( + translate_bfia_dag_to_dag(dag_schema).dag.tasks[waiting_task.f_task_name].inputs + ) + schedule_logger(job.f_job_id).info(f"task {waiting_task.f_task_name} dependent tasks:{dependent_tasks}") + for task_name in dependent_tasks: + dependent_task = tasks_group[task_name] + if dependent_task.f_status != TaskStatus.SUCCESS: + break + else: + scheduling_status_code = SchedulingStatusCode.HAVE_NEXT + status_code = cls.start_task( + job_id=waiting_task.f_job_id, + task_name=waiting_task.f_task_name, + task_id=waiting_task.f_task_id, + task_version=waiting_task.f_task_version + ) + if status_code == SchedulingStatusCode.FAILED: + schedule_logger(job.f_job_id).info(f"task status code: {status_code}") + scheduling_status_code = SchedulingStatusCode.FAILED + waiting_task.f_status = StatusSet.FAILED + BfiaFederatedScheduler.stop_job( + node_list=job.f_parties, + command_body={ + "job_id": job.f_job_id, + "task_name": waiting_task.f_task_name + }) + else: + schedule_logger(job.f_job_id).info("have cancel signal, pass start job tasks") + schedule_logger(job.f_job_id).info("finish scheduling job tasks") + return scheduling_status_code, tasks_group.values() + + @classmethod + def start_task(cls, job_id, task_name, task_id, task_version): + schedule_logger(job_id).info("try to start task {} {}".format(task_id, task_name)) + + tasks = BfiaScheduleJobSaver.query_task(task_id=task_id, task_name=task_name) + response_list = BfiaFederatedScheduler.start_task( + task_id=task_id, task_name=task_name, job_id=job_id, node_list=tasks[0].f_parties + ) + schedule_logger(job_id).info(f"start task response: {response_list}") + for resp in response_list.values(): + if resp["code"] != ReturnCode.SUCCESS: + return SchedulingStatusCode.FAILED + else: + # update scheduler task info to running + task_info = dict( + job_id=job_id, + task_name=task_name, + task_id=task_id, + status=TaskStatus.RUNNING, + task_version=task_version + ) + BfiaScheduleJobSaver.update_task_status( + task_info=task_info, + scheduler_status=True + ) + for task in tasks: + task_info.update({ + "role": task.f_role, + "party_id": task.f_party_id + }) + BfiaScheduleJobSaver.update_task_status( + task_info=task_info + + ) + return SchedulingStatusCode.SUCCESS + + @classmethod + def collect_task_of_all_party(cls, task, parties): + federated_response = BfiaFederatedScheduler.poll_task_all( + node_list=parties, + task_id=task.f_task_id + ) + for _party_id, party_response in federated_response.items(): + if party_response["code"] == ReturnCode.SUCCESS: + schedule_logger(task.f_job_id).info( + f"collect party id {_party_id} task {task.f_task_name} info: {party_response['data']}") + task_info = { + "job_id": task.f_job_id, + "task_id": task.f_task_id, + "task_version": task.f_task_version, + "role": "", + "party_id": _party_id, + "status": party_response["data"].get("status") + } + BfiaScheduleJobSaver.update_task_status(task_info=task_info) + + @classmethod + def get_federated_task_status(cls, task_name, job_id, task_id, task_version): + tasks_on_all_party = BfiaScheduleJobSaver.query_task(task_id=task_id, task_version=task_version) + tasks_party_status = [task.f_status for task in tasks_on_all_party] + status = BfiaTaskController.calculate_multi_party_task_status(tasks_party_status) + schedule_logger(job_id=job_id).info( + "task {} {} {} status is {}, calculate by task party status list: {}".format( + task_name, + task_id, + task_version, + status, + tasks_party_status + )) + return status diff --git a/python/fate_flow/adapter/bfia/settings.py b/python/fate_flow/adapter/bfia/settings.py new file mode 100644 index 000000000..7addde298 --- /dev/null +++ b/python/fate_flow/adapter/bfia/settings.py @@ -0,0 +1,20 @@ +from fate_flow.runtime.system_settings import PARTY_ID + +LOCAL_SITE_ID = PARTY_ID +STORAGE_NAME = "s3" +STORAGE_ADDRESS = "s3://127.0.0.1:9000?username=admin&password=12345678" +TRANSPORT = "127.0.0.1:9377" +SESSION_ID = "session_{}" +TOKEN = "session_{}" +FATE_CONTAINER_HOME = "/data/projects/fate/fate_flow" +CONTAINER_LOG_PATH = f"{FATE_CONTAINER_HOME}/logs" +CALLBACK_ADDRESS = "http://127.0.0.1:9380" +CALLBACK = f"{CALLBACK_ADDRESS}/v1/platform/schedule/task/callback" + + +VOLUME = { + # "/data/projects/fate/fate_flow/logs": { + # 'bind': "/data/projects/fate/fate_flow/logs", + # 'mode': 'rw' + # } +} diff --git a/python/component_plugins/__init__.py b/python/fate_flow/adapter/bfia/translator/__init__.py similarity index 100% rename from python/component_plugins/__init__.py rename to python/fate_flow/adapter/bfia/translator/__init__.py diff --git a/python/fate_flow/adapter/bfia/translator/component_spec.py b/python/fate_flow/adapter/bfia/translator/component_spec.py new file mode 100644 index 000000000..96638c6ad --- /dev/null +++ b/python/fate_flow/adapter/bfia/translator/component_spec.py @@ -0,0 +1,37 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pydantic import BaseModel +from typing import Optional, Dict, List, Literal + + +class DataSpec(BaseModel): + name: str + description: str + category: str + dataFormat: List[str] + + +class BFIAComponentSpec(BaseModel): + componentName: str + title: str + provider: str + version: str + description: str + roleList: List[Literal["guest", "host", "arbiter", "local"]] + desVersion: str + storageEngine: List[str] + inputParam: Optional[List[Dict]] + inputData: Optional[List[DataSpec]] + outputData: Optional[List[DataSpec]] diff --git a/python/fate_flow/adapter/bfia/translator/dsl_translator.py b/python/fate_flow/adapter/bfia/translator/dsl_translator.py new file mode 100644 index 000000000..7ab6e9dbe --- /dev/null +++ b/python/fate_flow/adapter/bfia/translator/dsl_translator.py @@ -0,0 +1,522 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import copy +from typing import Dict + +from .component_spec import BFIAComponentSpec + +from fate_flow.entity.spec.dag import ( + DAGSchema, + DAGSpec, + PartySpec, + RuntimeTaskOutputChannelSpec, + DataWarehouseChannelSpec, + PartyTaskSpec, + PartyTaskRefSpec, + TaskSpec, + RuntimeInputArtifacts, + JobConfSpec, + OutputArtifactSpec, + OutputArtifacts +) + +from ..utils.spec.job import ( + BFIADagSpec, + DagSchemaSpec, + ConfSpec, + InitiatorSpec, + RoleSpec, + JobCommonSpec, + JobParamsSpec, + TaskParamsSpec, + DagComponentSpec, + DataSpec, + DagSpec +) + + +DATASET_ID = "dataset_id" + + +class Translator(object): + @classmethod + def translate_dag_to_bfia_dag(cls, dag_schema: DAGSchema, component_specs: Dict[str, BFIAComponentSpec]): + flow_id = None + old_job_id = None + if dag_schema.dag.conf and dag_schema.dag.conf.extra: + flow_id = dag_schema.dag.conf.extra.get("flow_id", "") + old_job_id = dag_schema.dag.conf.extra.get("old_job_id", "") + + bfia_dag = BFIADagSpec( + flow_id=flow_id, + old_job_id=old_job_id, + config=cls.translate_dag_to_bfia_config(dag_schema.dag, dag_schema.schema_version), + dag=cls.translate_dag_to_bfia_tasks(dag_schema.dag, component_specs, dag_schema.schema_version) + ) + + return DagSchemaSpec( + kind=dag_schema.kind, + schema_version=dag_schema.schema_version, + dag=bfia_dag + ) + + @classmethod + def translate_bfia_dag_to_dag(cls, bfia_dag_schema: DagSchemaSpec, component_specs: Dict[str, BFIAComponentSpec]): + translated_dag_buf = dict() + translated_dag_buf["schema_version"] = bfia_dag_schema.schema_version + + bfia_dag: BFIADagSpec = bfia_dag_schema.dag + translated_dag_buf["schema_version"] = bfia_dag_schema.schema_version + translated_dag_buf["kind"] = bfia_dag_schema.kind + + dag_spec_buf = dict() + # dag_spec_buf["initiator"] = (bfia_dag.config.initiator.role, bfia_dag.config.initiator.node_id) + dag_spec_buf["parties"] = cls.get_party_spec_from_bfia_dag(bfia_dag) + # dag_spec_buf["flow_id"] = bfia_dag.flow_id + # dag_spec_buf["old_job_id"] = bfia_dag.old_job_id + dag_spec_buf["conf"] = cls.translate_job_params_to_dag(bfia_dag) + dag_spec_buf["tasks"] = cls.translate_bfia_tasks_to_dag(bfia_dag, component_specs) + dag_spec_buf["party_tasks"] = cls.translate_party_tasks_to_dag(bfia_dag, component_specs, dag_spec_buf["tasks"]) + + if not dag_spec_buf["conf"].extra: + dag_spec_buf["conf"].extra = dict() + dag_spec_buf["conf"].extra["flow_id"] = bfia_dag.flow_id + dag_spec_buf["conf"].extra["old_job_id"] = bfia_dag.old_job_id + dag_spec_buf["conf"].extra["initiator"] = dict( + role=bfia_dag.config.initiator.role, + party_id=bfia_dag.config.initiator.node_id + ) + + translated_dag_buf["dag"] = dag_spec_buf + return DAGSchema(**translated_dag_buf) + + @classmethod + def translate_job_params_to_dag(cls, bfia_dag: BFIADagSpec): + job_conf = JobConfSpec(**bfia_dag.config.job_params.common.dict(exclude_defaults=True)) + return job_conf + + @classmethod + def get_party_spec_from_bfia_dag(cls, bfia_dag: BFIADagSpec): + parties = [] + + for role, party_id_list in iter(bfia_dag.config.role): + if not party_id_list: + continue + + parties.append(PartySpec(role=role, party_id=party_id_list)) + + return parties + + @classmethod + def translate_bfia_tasks_to_dag(cls, bfia_dag: BFIADagSpec, component_specs: Dict[str, BFIAComponentSpec]): + tasks = dict() + + common_params = bfia_dag.config.task_params.common + for bfia_task in bfia_dag.dag.components: + task_spec = TaskSpec(component_ref=bfia_task.componentName) + task_name = bfia_task.name + tasks[task_name] = task_spec + + if common_params and task_name in common_params: + task_spec.parameters = common_params[task_name] + + dependencies = set() + for input_desc in bfia_task.input: + dependencies.add(input_desc.key.split(".", -1)[0]) + task_spec.dependent_tasks = list(dependencies) + + conf = dict() + conf["provider"] = bfia_task.provider + conf["version"] = bfia_task.version + + component_spec = component_specs[bfia_task.componentName] + support_roles = set(component_spec.roleList) + parties = [] + all_parties = cls.get_party_spec_from_bfia_dag(bfia_dag) + + for party in all_parties: + if party.role in support_roles: + parties.append(party) + + task_spec.parties = parties + + if bfia_task.input: + input_keys = dict() + for input_spec in component_spec.inputData: + input_type = get_source_type(input_spec.category) + input_name = input_spec.name + if input_type not in input_keys: + input_keys[input_type] = [] + + input_keys[input_type].append(input_name) + + inputs = dict() + for input_desc in bfia_task.input: + input_type = get_source_type(input_desc.type) + + if input_type not in inputs: + inputs[input_type] = dict() + + producer_task, output_artifact_key = input_desc.key.split(".", -1) + + input_spec = RuntimeTaskOutputChannelSpec(producer_task=producer_task, + output_artifact_key=output_artifact_key, + output_artifact_type_alias=input_desc.type) + input_name = input_keys[input_type].pop(0) + + # TODO: bifa does not support multiple inputs yet + inputs[input_type][input_name] = dict(task_output_artifact=input_spec) + + task_spec.inputs = RuntimeInputArtifacts(**inputs) + + if bfia_task.output: + output_keys = dict() + for output_spec in component_spec.outputData: + output_name = output_spec.name + output_type = get_source_type(output_spec.category) + + if output_type not in output_keys: + output_keys[output_type] = [] + + output_keys[output_type].append(output_name) + + outputs = dict() + for output_dict in bfia_task.output: + output_alias = output_dict.key + output_type_alias = output_dict.type + + output_type = get_source_type(output_type_alias) + if output_type not in outputs: + outputs[output_type] = dict() + + output_name = output_keys[output_type].pop(0) + + outputs[output_type][output_name] = OutputArtifactSpec( + output_artifact_key_alias=output_alias, + output_artifact_type_alias=output_type_alias, + ) + + task_spec.outputs = OutputArtifacts(**outputs) + + task_spec.conf = conf + tasks[task_name] = task_spec + + return cls.add_dataset_from_party_task(tasks, bfia_dag, component_specs) + + @classmethod + def add_dataset_from_party_task( + cls, + tasks: Dict[str, TaskSpec], + bfia_dag: BFIADagSpec, + component_specs: Dict[str, BFIAComponentSpec] + ): + if bfia_dag.config and bfia_dag.config.task_params: + for role, role_params in iter(bfia_dag.config.task_params): + if role == "common" or not role_params: + continue + + for party_str, party_task_params in role_params.items(): + party_id_indexes = list(map(int, party_str.split("|", -1))) + party_id_list = [getattr(bfia_dag.config.role, role)[party_id] for party_id in party_id_indexes] + + for task_name, params in party_task_params.items(): + if DATASET_ID not in params: + continue + + component_ref = None + if task_name not in tasks: + for component in bfia_dag.dag.components: + if component.name == "task_name": + component_ref = component.componentName + + if not component_ref: + raise ValueError(f"Can not find task={task_name}") + + tasks[task_name] = TaskSpec(component_ref=component_ref) + else: + component_ref = tasks[task_name].component_ref + + input_name = component_specs[component_ref].inputData[0].name + data_warehouse_spec = DataWarehouseChannelSpec( + dataset_id=params[DATASET_ID], + parties=[PartySpec(role=role, party_id=party_id_list)] + ) + if not tasks[task_name].inputs: + tasks[task_name].inputs = RuntimeInputArtifacts(data=dict()) + if input_name not in tasks[task_name].inputs.data: + tasks[task_name].inputs.data[input_name] = dict(data_warehouse=data_warehouse_spec) + elif "data_warehouse" not in tasks[task_name].inputs.data[input_name]: + tasks[task_name].inputs.data[input_name]["data_warehouse"] = data_warehouse_spec + elif not isinstance(tasks[task_name].inputs.data[input_name]["data_warehouse"], list): + pre_spec = tasks[task_name].inputs.data[input_name]["data_warehouse"] + tasks[task_name].inputs.data[input_name]["data_warehouse"] = [ + pre_spec, data_warehouse_spec + ] + else: + tasks[task_name].inputs.data[input_name]["data_warehouse"].append(data_warehouse_spec) + + return tasks + + @classmethod + def translate_party_tasks_to_dag(cls, + bfia_dag: BFIADagSpec, + component_specs: Dict[str, BFIAComponentSpec], + tasks: Dict[str, TaskSpec] + ): + party_tasks = dict() + if bfia_dag.config and bfia_dag.config.job_params: + for role, role_config in iter(bfia_dag.config.job_params): + if role == "common" or not role_config: + continue + + role_task_params = getattr(getattr(bfia_dag.config, "task_params", {}), role, {}) + + for party_str, party_config in role_config.items(): + party_id_indexes = list(map(int, party_str.split("|", -1))) + party_id_list = [getattr(bfia_dag.config.role, role)[party_id] for party_id in party_id_indexes] + party_task = PartyTaskSpec() + party_task.parties = [PartySpec(role=role, party_id=party_id_list)] + party_task.conf = party_config + + if role_task_params and party_str in role_task_params: + party_task_params = role_task_params[party_str] + party_task.tasks = cls.get_party_task_params(party_task_params) + + site_name = "_".join(map(str, [role] + party_id_list)) + party_tasks[site_name] = party_task + + if bfia_dag.config and bfia_dag.config.task_params: + for role, role_params in iter(bfia_dag.config.task_params): + if role == "common" or not role_params: + continue + + for party_str, party_task_params in role_params.items(): + party_id_indexes = list(map(int, party_str.split("|", -1))) + party_id_list = [getattr(bfia_dag.config.role, role)[party_id] for party_id in party_id_indexes] + site_name = "_".join(map(str, [role] + party_id_list)) + + if site_name in party_tasks: + continue + + party_task = PartyTaskSpec() + party_task.parties = [PartySpec(role=role, party_id=party_id_list)] + party_task.tasks = cls.get_party_task_params(party_task_params) + + party_tasks[site_name] = party_task + + return party_tasks + + @classmethod + def get_party_task_params(cls, party_task_params): + party_task_specs = dict() + + for task_name, params in party_task_params.items(): + task_spec = PartyTaskRefSpec() + params = copy.deepcopy(params) + if DATASET_ID in params: + params.pop(DATASET_ID) + + if params: + task_spec.parameters = params + + return party_task_specs + + @classmethod + def translate_dag_to_bfia_config(cls, dag: DAGSpec, schema_version: str): + bfia_conf_buf = dict(version=schema_version) + + if dag.conf and dag.conf.extra and "initiator" in dag.conf.extra: + bfia_conf_buf["initiator"] = InitiatorSpec( + role=dag.conf.extra["initiator"]["role"], + node_id=dag.conf.extra["initiator"]["party_id"] + ) + + role_spec = RoleSpec() + for party_spec in dag.parties: + role = party_spec.role + party_id_list = party_spec.party_id + setattr(role_spec, role, party_id_list) + + bfia_conf_buf["role"] = role_spec + + job_params = JobParamsSpec() + if dag.conf: + job_params.common = JobCommonSpec(**dag.conf.dict(exclude_defaults=True)) + + if dag.party_tasks: + parties_conf = dict() + for site_name, party_task in dag.party_tasks.items(): + if party_task.conf: + for party_spec in party_task.parties: + party_str = "|".join(map(str, + [getattr(role_spec, party_spec.role).index(party_id) for party_id in party_spec.party_id] + )) + + if party_spec.role not in parties_conf: + parties_conf[party_spec.role] = dict() + + parties_conf[party_spec.role][party_str] = party_task.conf + + for role, conf in parties_conf.items(): + setattr(job_params, role, conf) + + bfia_conf_buf["job_params"] = job_params + + task_params = TaskParamsSpec() + if dag.tasks: + common_params = dict() + for task_name, task_spec in dag.tasks.items(): + if task_spec.parameters: + common_params[task_name] = task_spec.parameters + + if common_params: + task_params.common = common_params + + if dag.party_tasks: + party_task_params = dict() + for site_name, party_task in dag.party_tasks.items(): + if not party_task.tasks: + continue + + party_conf = dict() + role = party_task.parties[0].role + party_id_list = party_task.parties[0].party_id + party_id_indexes = [getattr(role_spec, role).index(party_id) for party_id in party_id_list] + party_str = "|".join(map(str, party_id_indexes)) + + for task_name, party_task_spec in party_task.tasks.items(): + party_conf[task_name] = dict() + if party_task_spec.parameters: + party_conf[task_name].update(party_task_spec.parameters) + + if role not in party_task_params: + party_task_params[role] = dict() + party_task_params[role][party_str] = party_conf + + + for role, conf in party_task_params.items(): + setattr(task_params, role, conf) + + if dag.tasks: + for task_name, task_spec in dag.tasks.items(): + if not task_spec.inputs or not task_spec.inputs.data: + continue + + parties = task_spec.parties if task_spec.parties else dag.parties + for _, input_artifact_specs in task_spec.inputs.data.items(): + for input_artifact_key, input_spec_list in input_artifact_specs.items(): + if not isinstance(input_spec_list, list): + input_spec_list = [input_spec_list] + for input_spec in input_spec_list: + if not isinstance(input_spec, DataWarehouseChannelSpec): + continue + input_parties = input_spec.parties if input_spec.parties else parties + for party_spec in input_parties: + party_str = "|".join(map(str, + [getattr(role_spec, party_spec.role).index(party_id) for + party_id in party_spec.party_id] + )) + input_dict = { + party_str: {task_name: dict(dataset_id=input_spec.dataset_id)}} + if not getattr(task_params, party_spec.role): + setattr(task_params, party_spec.role, input_dict) + else: + getattr(task_params, party_spec.role).update(input_dict) + + bfia_conf_buf["task_params"] = task_params + + return ConfSpec(**bfia_conf_buf) + + @classmethod + def translate_dag_to_bfia_tasks(cls, dag: DAGSpec, component_specs: Dict[str, BFIAComponentSpec], schema_version): + bfia_dag_buf = dict(version=schema_version) + + tasks = [] + if dag.tasks: + for task_name, task_spec in dag.tasks.items(): + bfia_task_spec = DagComponentSpec( + provider=task_spec.conf["provider"], + version=task_spec.conf["version"], + name=task_name, + componentName=task_spec.component_ref + ) + + component_spec = component_specs[task_spec.component_ref] + inputs = [] + if task_spec.inputs: + for input_desc in component_spec.inputData: + input_type = get_source_type(input_desc.category) + input_key = input_desc.name + + input_artifact_specs = getattr(task_spec.inputs, input_type, {}) + if not input_artifact_specs or input_key not in input_artifact_specs: + continue + + input_spec = input_artifact_specs[input_key] + + if "task_output_artifact" not in input_spec: + continue + producer_task = input_spec["task_output_artifact"].producer_task + output_artifact_key = input_spec["task_output_artifact"].output_artifact_key + type_alias = input_spec["task_output_artifact"].output_artifact_type_alias + + if type_alias is None: + type_alias= getattr(dag.tasks[producer_task].outputs, input_type)[output_artifact_key].output_artifact_type_alias + + inputs.append(DataSpec(type=type_alias, key=".".join([producer_task, output_artifact_key]))) + + bfia_task_spec.input = inputs + + outputs = [] + if task_spec.outputs: + for output_desc in component_spec.outputData: + output_type = get_source_type(output_desc.category) + output_key = output_desc.name + + output_artifacts = getattr(task_spec.outputs, output_type, {}) + if not output_artifacts or output_key not in output_artifacts: + continue + + output_spec: OutputArtifactSpec = output_artifacts[output_key] + + if not output_spec: + continue + + outputs.append(DataSpec(type=output_spec.output_artifact_type_alias, + key=output_spec.output_artifact_key_alias)) + + bfia_task_spec.output = outputs + + tasks.append(bfia_task_spec) + + if tasks: + bfia_dag_buf["components"] = tasks + + return DagSpec(**bfia_dag_buf) + + +def get_source_type(type_keyword): + data_keywords = ["data", "dataset", "training_set", "test_set", "validate_set"] + model_keywords = ["model"] + for data_keyword in data_keywords: + if data_keyword in type_keyword: + return "data" + + for model_keyword in model_keywords: + if model_keyword in type_keyword: + return "model" + + return "metric" diff --git a/python/fate_flow/adapter/bfia/utils/api_utils.py b/python/fate_flow/adapter/bfia/utils/api_utils.py new file mode 100644 index 000000000..dd590c161 --- /dev/null +++ b/python/fate_flow/adapter/bfia/utils/api_utils.py @@ -0,0 +1,24 @@ +from flask import jsonify + +from fate_flow.adapter.bfia.utils.entity.code import ReturnCode +from fate_flow.utils.api_utils import API + + +class BfiaAPI(API): + class Output: + @staticmethod + def json(code=ReturnCode.SUCCESS, msg='success', data=None, **kwargs): + result_dict = { + "code": code, + "msg": msg, + "data": data, + } + + response = {} + for key, value in result_dict.items(): + if value is not None: + response[key] = value + # extra resp + for key, value in kwargs.items(): + response[key] = value + return jsonify(response) \ No newline at end of file diff --git a/python/fate_flow/adapter/bfia/utils/entity/__init__.py b/python/fate_flow/adapter/bfia/utils/entity/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/adapter/bfia/utils/entity/code.py b/python/fate_flow/adapter/bfia/utils/entity/code.py new file mode 100644 index 000000000..a134690fc --- /dev/null +++ b/python/fate_flow/adapter/bfia/utils/entity/code.py @@ -0,0 +1,3 @@ +class ReturnCode: + SUCCESS = 0 + FAILED = 1 diff --git a/python/fate_flow/adapter/bfia/utils/entity/status.py b/python/fate_flow/adapter/bfia/utils/entity/status.py new file mode 100644 index 000000000..b2759226e --- /dev/null +++ b/python/fate_flow/adapter/bfia/utils/entity/status.py @@ -0,0 +1,58 @@ +from fate_flow.entity.types import BaseStateTransitionRule, BaseStatus + + +class StatusSet(BaseStatus): + PENDING = "PENDING" + READY = 'READY' + RUNNING = "RUNNING" + FINISHED = "FINISHED" + REJECTED = "REJECTED" + SUCCESS = "SUCCESS" + FAILED = "FAILED" + + @classmethod + def get_level(cls, status): + return dict(zip(cls.status_list(), range(len(cls.status_list())))).get(status, None) + + +class JobStatus(BaseStatus): + PENDING = StatusSet.PENDING + READY = StatusSet.READY + REJECTED = StatusSet.REJECTED + RUNNING = StatusSet.RUNNING + FINISHED = StatusSet.FINISHED + + class StateTransitionRule(BaseStateTransitionRule): + RULES = { + StatusSet.PENDING: [StatusSet.READY, StatusSet.REJECTED], + StatusSet.READY: [StatusSet.RUNNING, StatusSet.FINISHED], + StatusSet.RUNNING: [StatusSet.FINISHED], + StatusSet.FINISHED: [] + } + + +class TaskStatus(BaseStatus): + PENDING = StatusSet.PENDING + READY = StatusSet.READY + RUNNING = StatusSet.RUNNING + SUCCESS = StatusSet.SUCCESS + FAILED = StatusSet.FAILED + + class StateTransitionRule(BaseStateTransitionRule): + RULES = { + StatusSet.PENDING: [StatusSet.READY, StatusSet.RUNNING, StatusSet.SUCCESS, StatusSet.FAILED], + StatusSet.READY: [StatusSet.RUNNING, StatusSet.FAILED, StatusSet.SUCCESS], + StatusSet.RUNNING: [StatusSet.SUCCESS, StatusSet.FAILED], + StatusSet.FAILED: [], + StatusSet.SUCCESS: [], + } + + +class EndStatus(BaseStatus): + FAILED = StatusSet.FAILED + FINISHED = StatusSet.FINISHED + SUCCESS = StatusSet.SUCCESS + + +class InterruptStatus(BaseStatus): + FAILED = StatusSet.FAILED diff --git a/python/fate_flow/adapter/bfia/utils/spec/artifact.py b/python/fate_flow/adapter/bfia/utils/spec/artifact.py new file mode 100644 index 000000000..7ff861d58 --- /dev/null +++ b/python/fate_flow/adapter/bfia/utils/spec/artifact.py @@ -0,0 +1,22 @@ +from typing import Optional, Dict + +from pydantic import BaseModel + + +class S3Address(BaseModel): + url: str + + +class Engine(BaseModel): + name: str = "s3" + address: S3Address + + +class ArtifactAddress(BaseModel): + name: str + namespace: str + + +class Artifact(BaseModel): + input: Dict[str, ArtifactAddress] + output: Dict[str, ArtifactAddress] diff --git a/python/fate_flow/adapter/bfia/utils/spec/job.py b/python/fate_flow/adapter/bfia/utils/spec/job.py new file mode 100644 index 000000000..6f0254dcb --- /dev/null +++ b/python/fate_flow/adapter/bfia/utils/spec/job.py @@ -0,0 +1,72 @@ +from typing import Optional, Dict, List + +from pydantic import BaseModel + + +class InitiatorSpec(BaseModel): + role: str + node_id: str + + +class RoleSpec(BaseModel): + guest: Optional[List[str]] + host: Optional[List[str]] + arbiter: Optional[List[str]] + + +class JobCommonSpec(BaseModel): + sync_type: Optional[str] = "poll" + + +class JobParamsSpec(BaseModel): + common: Optional[JobCommonSpec] + guest: Optional[Dict] + host: Optional[Dict] + arbiter: Optional[Dict] + + +class TaskParamsSpec(BaseModel): + common: Optional[Dict] + guest: Optional[Dict] + host: Optional[Dict] + arbiter: Optional[Dict] + + +class ConfSpec(BaseModel): + initiator: InitiatorSpec + role: RoleSpec + job_params: JobParamsSpec + task_params: TaskParamsSpec + version: str + + +class DataSpec(BaseModel): + key: str + type: str + + +class DagComponentSpec(BaseModel): + name: str + componentName: str + provider: str + version: str + input: Optional[List[DataSpec]] = [] + output: Optional[List[DataSpec]] = [] + + +class DagSpec(BaseModel): + components: List[DagComponentSpec] + version: str + + +class BFIADagSpec(BaseModel): + config: ConfSpec + dag: DagSpec + flow_id: Optional[str] + old_job_id: Optional[str] + + +class DagSchemaSpec(BaseModel): + dag: BFIADagSpec + schema_version: str + kind: str = "bfia" diff --git a/python/fate_flow/adapter/bfia/utils/spec/task.py b/python/fate_flow/adapter/bfia/utils/spec/task.py new file mode 100644 index 000000000..6ef685c55 --- /dev/null +++ b/python/fate_flow/adapter/bfia/utils/spec/task.py @@ -0,0 +1,43 @@ +from typing import Optional, Dict + +from pydantic import BaseModel + +from fate_flow.adapter.bfia.utils.spec.artifact import ArtifactAddress + + +class RuntimeConf(BaseModel): + name: str + parameter: Dict = {} + input: Optional[Dict[str, ArtifactAddress]] + output: Optional[Dict[str, ArtifactAddress]] + + +class RuntimeComponent(BaseModel): + component: RuntimeConf + + +class LogPath(BaseModel): + path: str + + +class Config(BaseModel): + task_id: str + trace_id: Optional[str] + session_id: str = "" + token: str = "" + inst_id: Dict + node_id: Dict + log: Optional[LogPath] + self_role: str + + +class SystemConf(BaseModel): + storage: str + transport: str + callback: str + + +class TaskRuntimeEnv(BaseModel): + runtime: RuntimeComponent + config: Config + system: SystemConf diff --git a/python/fate_flow/adapter/bfia/wheels/__init__.py b/python/fate_flow/adapter/bfia/wheels/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/adapter/bfia/wheels/federated.py b/python/fate_flow/adapter/bfia/wheels/federated.py new file mode 100644 index 000000000..a545aea88 --- /dev/null +++ b/python/fate_flow/adapter/bfia/wheels/federated.py @@ -0,0 +1,62 @@ +from fate_flow.adapter.bfia import BfiaRuntimeConfig + + +class BfiaFederatedScheduler: + # job + @classmethod + def create_job(cls, node_list, command_body): + return BfiaRuntimeConfig.SCHEDULE_CLIENT.federated.create_job( + node_list, command_body=command_body + ) + + @classmethod + def start_job(cls, node_list, job_id): + command_body = { + "job_id": job_id + } + return BfiaRuntimeConfig.SCHEDULE_CLIENT.federated.start_job( + node_list, command_body=command_body + ) + + @classmethod + def stop_job(cls, node_list, command_body): + return BfiaRuntimeConfig.SCHEDULE_CLIENT.federated.stop_job( + node_list, command_body=command_body + ) + + @classmethod + def poll_task_all(cls, node_list, task_id): + command_body = { + "task_id": task_id, + "role": "" + } + return BfiaRuntimeConfig.SCHEDULE_CLIENT.federated.poll_task(node_list, command_body=command_body) + + @classmethod + def start_task(cls, node_list, job_id, task_id, task_name): + command_body = { + "task_id": task_id, + "task_name": task_name, + "job_id": job_id + } + return BfiaRuntimeConfig.SCHEDULE_CLIENT.federated.start_task(node_list, command_body=command_body) + + # scheduler + @classmethod + def request_create_job(cls, party_id, command_body): + return BfiaRuntimeConfig.SCHEDULE_CLIENT.scheduler.create_job(party_id, command_body) + + @classmethod + def request_audit_confirm(cls, party_id, command_body): + return BfiaRuntimeConfig.SCHEDULE_CLIENT.scheduler.audit_confirm(party_id, command_body) + + @classmethod + def request_stop_job(cls, party_id, job_id): + command_body = { + "job_id": job_id + } + return BfiaRuntimeConfig.SCHEDULE_CLIENT.scheduler.stop_job(party_id, command_body) + + @classmethod + def request_report_task(cls, party_id, command_body): + return BfiaRuntimeConfig.SCHEDULE_CLIENT.scheduler.report_task(party_id, command_body) diff --git a/python/fate_flow/adapter/bfia/wheels/job.py b/python/fate_flow/adapter/bfia/wheels/job.py new file mode 100644 index 000000000..8254450ce --- /dev/null +++ b/python/fate_flow/adapter/bfia/wheels/job.py @@ -0,0 +1,222 @@ +from copy import deepcopy + +from fate_flow.adapter.bfia.wheels.parser import get_dag_parser + +from fate_flow.adapter.bfia.settings import LOCAL_SITE_ID as PARTY_ID +from fate_flow.adapter.bfia.utils.entity.code import ReturnCode +from fate_flow.adapter.bfia.utils.entity.status import TaskStatus, JobStatus, EndStatus +from fate_flow.adapter.bfia.utils.spec.job import DagSchemaSpec +from fate_flow.adapter.bfia.wheels.federated import BfiaFederatedScheduler +from fate_flow.adapter.bfia.wheels.task import BfiaTaskController +from fate_flow.db import Job +from fate_flow.adapter.bfia.wheels.saver import BfiaJobSaver as JobSaver +from fate_flow.utils.base_utils import current_timestamp +from fate_flow.utils.job_utils import save_job_dag, generate_job_id +from fate_flow.utils.log_utils import schedule_logger + + +class BfiaJobController(object): + @classmethod + def request_create_job(cls, dag, config, flow_id, old_job_id): + job_id = generate_job_id() + job_info = { + "dag": dag, + "config": config, + "flow_id": flow_id, + "old_job_id": old_job_id, + "job_id": job_id + } + schedule_logger(job_id).info("start request create job") + resp = BfiaFederatedScheduler.request_create_job( + party_id=PARTY_ID, + command_body=job_info + ) + schedule_logger(job_id).info(f"response: {resp}") + if resp and isinstance(resp, dict) and resp.get("code") == ReturnCode.SUCCESS: + save_job_dag(job_id=job_id, dag=job_info) + return job_id + else: + raise RuntimeError(resp) + + @classmethod + def query_job_status(cls, **kwargs): + all_status = {} + status = {} + jobs = JobSaver.query_job(**kwargs) + for job in jobs: + if job.f_job_id not in all_status: + all_status[job.f_job_id] = [job.f_status] + + for job_id, status_list in all_status.items(): + status[job_id] = cls.calculate_multi_party_job_status(status_list) + + return status + + @classmethod + def request_stop_job(cls, job_id): + jobs = JobSaver.query_job( + job_id=job_id + ) + if not jobs: + raise RuntimeError("No found jobs") + + schedule_logger(job_id).info(f"stop job on this party") + cls.stop_local_jobs(job_id=job_id) + + schedule_logger(job_id).info(f"request stop job") + response = BfiaFederatedScheduler.request_stop_job( + party_id=jobs[0].f_scheduler_party_id, + job_id=job_id + ) + schedule_logger(job_id).info(f"stop job response: {response}") + return response + + @classmethod + def create_local_jobs(cls, job_id, dag): + schedule_logger(job_id).info(f"start create job {job_id}") + schedule_logger(job_id).info(f"job dag schema: {dag}") + dag_schema = DagSchemaSpec(**dag) + for role, node_id_list in dag_schema.dag.config.role.dict().items(): + if node_id_list and PARTY_ID in node_id_list: + cls.create_local_job(job_id, role, PARTY_ID, dag_schema) + schedule_logger(job_id).info(f"create job {job_id} success") + + @classmethod + def create_local_job(cls, job_id, role, node_id, dag_schema: DagSchemaSpec): + schedule_logger(job_id).info(f"create job {job_id} role {role}") + job = Job() + job.f_flow_id = dag_schema.dag.flow_id + job.f_protocol = dag_schema.kind + job.f_job_id = job_id + job.f_role = role + job.f_party_id = node_id + job.f_dag = dag_schema.dict() + job.f_progress = 0 + job.f_parties = cls.get_job_parties(dag_schema) + job.f_initiator_party_id = dag_schema.dag.config.initiator.node_id + job.f_scheduler_party_id = dag_schema.dag.config.initiator.node_id + job.f_status = JobStatus.READY + job.f_model_id = job_id + job.f_model_version = "0" + JobSaver.create_job(job_info=job.to_human_model_dict()) + + @classmethod + def start_job(cls, job_id): + schedule_logger(job_id).info(f"try to start job") + job_info = { + "job_id": job_id, + "start_time": current_timestamp() + } + cls.update_job_info(job_info=job_info, callback=cls.update_job) + job_info["status"] = JobStatus.RUNNING + cls.update_job_info(job_info=job_info, callback=cls.update_job_status) + schedule_logger(job_id).info(f"start job on status {job_info.get('status')}") + + @classmethod + def update_job_info(cls, job_info, callback): + info = deepcopy(job_info) + if "role" not in job_info or "party_id" not in job_info: + job_list = JobSaver.query_job(job_id=job_info.get("job_id")) + for job in job_list: + info["role"] = job.f_role + info["party_id"] = job.f_party_id + callback(info) + else: + callback(info) + + @classmethod + def update_job_status(cls, job_info): + update_status = JobSaver.update_job_status(job_info=job_info) + if update_status and EndStatus.contains(job_info.get("status")): + pass + + @classmethod + def update_job(cls, job_info): + return JobSaver.update_job(job_info=job_info) + + @staticmethod + def get_job_parties(dag_schema: DagSchemaSpec): + return set( + value for values_list in dag_schema.dag.config.role.dict().values() if values_list for value in values_list + ) + + @classmethod + def stop_local_jobs(cls, job_id, task_name=None): + jobs = JobSaver.query_job( + job_id=job_id + ) + + for job in jobs: + cls.stop_job(job=job, task_name=task_name) + + @classmethod + def stop_job(cls, job, task_name): + stop_status = TaskStatus.FAILED + schedule_logger(job_id=job.f_job_id).info("start stop job on local") + # get tasks + if task_name: + tasks = JobSaver.query_task( + job_id=job.f_job_id, role=job.f_role, party_id=job.f_party_id, + only_latest=True, reverse=True, task_name=task_name + ) + else: + tasks = JobSaver.query_task( + job_id=job.f_job_id, role=job.f_role, party_id=job.f_party_id, + only_latest=True, reverse=True + ) + update_job = False + # stop tasks + for task in tasks: + if task.f_party_status == TaskStatus.FAILED: + update_job = True + # if task.f_status in [TaskStatus.SUCCESS, TaskStatus.PENDING, TaskStatus.READY]: + # continue + schedule_logger(job_id=job.f_job_id).info(f"[stop]start to kill task {task.f_task_name} " + f"status {task.f_party_status}") + status = BfiaTaskController.stop_task(task, stop_status=stop_status) + schedule_logger(job_id=job.f_job_id).info(f"[stop]Kill {task.f_task_name} task completed: {status}") + + # update job status + if update_job or cls.calculate_job_is_finished(job): + BfiaJobController.update_job_status({ + "job_id": job.f_job_id, + "role": job.f_role, + "party_id": job.f_party_id, + "status": JobStatus.FINISHED + }) + + @classmethod + def calculate_job_is_finished(cls, job): + schedule_logger(job.f_job_id).info("start to calculate job status") + dag_schema = DagSchemaSpec.parse_obj(job.f_dag) + job_parser = get_dag_parser(dag_schema) + task_list = job_parser.party_topological_sort(role=job.f_role, party_id=job.f_party_id) + waiting_list = [] + for name in task_list: + tasks = JobSaver.query_task( + job_id=job.f_job_id, role=job.f_role, party_id=job.f_party_id, + only_latest=True, reverse=True, task_name=name + ) + if not tasks: + waiting_list.append(name) + if waiting_list: + schedule_logger(job.f_job_id).info(f"task {waiting_list} is waiting to run") + return False + return True + + @classmethod + def calculate_multi_party_job_status(cls, party_status): + tmp_status_set = set(party_status) + if len(tmp_status_set) == 1: + return tmp_status_set.pop() + else: + if JobStatus.REJECTED in tmp_status_set: + return JobStatus.REJECTED + if JobStatus.FINISHED in tmp_status_set: + return JobStatus.FINISHED + if JobStatus.RUNNING in tmp_status_set: + return JobStatus.RUNNING + if JobStatus.READY in tmp_status_set: + return JobStatus.READY + if JobStatus.PENDING in tmp_status_set: + return JobStatus.PENDING diff --git a/python/fate_flow/adapter/bfia/wheels/output.py b/python/fate_flow/adapter/bfia/wheels/output.py new file mode 100644 index 000000000..8e081a54b --- /dev/null +++ b/python/fate_flow/adapter/bfia/wheels/output.py @@ -0,0 +1,19 @@ +from fate_flow.adapter.bfia.db import ComponentOutput +from fate_flow.db.base_models import BaseModelOperate +from fate_flow.utils.wraps_utils import filter_parameters + + +class OutputMeta(BaseModelOperate): + @classmethod + def save(cls, **meta_info): + cls._create_entity(ComponentOutput, meta_info) + + @classmethod + @filter_parameters() + def query(cls, **kwargs): + return cls._query(ComponentOutput, **kwargs) + + @classmethod + @filter_parameters() + def delete(cls, **kwargs): + return cls._delete(ComponentOutput, **kwargs) diff --git a/python/fate_flow/adapter/bfia/wheels/parser.py b/python/fate_flow/adapter/bfia/wheels/parser.py new file mode 100644 index 000000000..309016853 --- /dev/null +++ b/python/fate_flow/adapter/bfia/wheels/parser.py @@ -0,0 +1,129 @@ +from fate_flow.adapter.bfia.settings import TRANSPORT, SESSION_ID, TOKEN, STORAGE_ADDRESS, STORAGE_NAME, \ + CALLBACK, CONTAINER_LOG_PATH +from fate_flow.adapter.bfia.translator.component_spec import BFIAComponentSpec +from fate_flow.adapter.bfia.translator.dsl_translator import Translator +from fate_flow.adapter.bfia.utils.spec.artifact import ArtifactAddress, Engine, S3Address +from fate_flow.adapter.bfia.utils.spec.job import DagSchemaSpec +from fate_flow.adapter.bfia.utils.spec.task import TaskRuntimeEnv, RuntimeComponent, RuntimeConf, Config, SystemConf, \ + LogPath +from fate_flow.adapter.bfia.wheels.output import OutputMeta +from fate_flow.entity.spec.dag import DataWarehouseChannelSpec, RuntimeTaskOutputChannelSpec, OutputArtifactSpec +from fate_flow.controller.parser import TaskParser, JobParser +from fate_flow.manager.service.provider_manager import ProviderManager + + +class BfiaTaskParser(TaskParser): + @property + def task_parameters(self): + return TaskRuntimeEnv( + runtime=RuntimeComponent(component=RuntimeConf( + name=self.task_node.component_ref, + parameter=self.input_parameters, + input=self.runtime_inputs, + output=self.runtime_outputs + )), + config=Config( + inst_id=self.node_id, + node_id=self.node_id, + log=LogPath(path=CONTAINER_LOG_PATH), + self_role=f"{self.role}.{self.role_index}", + task_id=self.task_id, + session_id=SESSION_ID.format(self.job_id), + token=TOKEN.format(self.job_id) + ), + system=SystemConf(storage=STORAGE_ADDRESS, transport=TRANSPORT, callback=CALLBACK) + ) + + @property + def role_index(self): + _nodes = {} + for party in self.parties: + if party.role not in _nodes: + _nodes[party.role] = party.party_id + return _nodes[self.role].index(self.party_id) + + @property + def node_id(self): + _nodes = {} + nodes = {} + for party in self.parties: + if party.role not in _nodes: + _nodes[party.role] = party.party_id + for _k, _v_list in _nodes.items(): + for _n, _v in enumerate(_v_list): + nodes[f"{_k}.{_n}"] = _v + return nodes + + @property + def runtime_inputs(self): + inputs = {} + for type, upstream_input in self.task_node.upstream_inputs.items(): + for key, channel in upstream_input.items(): + if isinstance(channel, DataWarehouseChannelSpec): + if channel.dataset_id: + namespace, name = channel.dataset_id.split("#") + else: + namespace, name = channel.namespace, channel.name + inputs[key] = ArtifactAddress(name=name, namespace=namespace) + elif isinstance(channel, RuntimeTaskOutputChannelSpec): + metas = OutputMeta.query( + job_id=self.job_id, role=self.role, party_id=self.party_id, + task_name=channel.producer_task, key=channel.output_artifact_key, + type=channel.output_artifact_type_alias + ) + if metas: + meta = metas[0] + inputs[key] = ArtifactAddress(**meta.f_address) + return inputs + + @property + def runtime_outputs(self): + outputs = {} + for type, output in self.task_node.outputs.items(): + for key, channel in output.items(): + if isinstance(channel, OutputArtifactSpec): + if self.role in [party.role for party in channel.parties]: + outputs[key] = self.create_output_address(channel) + return outputs + + def create_output_address(self, channel: OutputArtifactSpec): + namespace = f"{self.task_id}" + name = f"{self.role}-{self.party_id}-{channel.output_artifact_type_alias}-{channel.output_artifact_key_alias}" + address = ArtifactAddress(name=namespace, namespace=name) + engine = Engine(name=STORAGE_NAME, address=S3Address(url=STORAGE_ADDRESS)) + meta = dict( + job_id=self.job_id, role=self.role, node_id=self.party_id, task_name=self.task_name, + component=self.task_node.component_ref, task_id=self.task_id, + type=channel.output_artifact_type_alias, key=channel.output_artifact_key_alias, + engine=engine.dict(), address=address.dict() + ) + try: + OutputMeta.save(**meta) + except Exception as e: + raise Exception(f"{e}, {meta}") + return address + + @property + def provider(self): + provider_name = self.task_runtime_conf.get("provider") + version = self.task_runtime_conf.get("version") + device = "docker" + self._provider = ProviderManager.generate_provider_name(provider_name, version, device) + return self._provider + + +class BfiaDagParser(JobParser): + @property + def task_parser(self): + return BfiaTaskParser + + +def get_dag_parser(dag: DagSchemaSpec) -> BfiaDagParser: + return BfiaDagParser(translate_bfia_dag_to_dag(dag)) + + +def translate_bfia_dag_to_dag(dag): + components_desc = {} + for name, desc in ProviderManager.query_component_description(protocol=dag.kind).items(): + components_desc[name] = BFIAComponentSpec.parse_obj(desc) + return Translator.translate_bfia_dag_to_dag(dag, components_desc) diff --git a/python/fate_flow/adapter/bfia/wheels/saver.py b/python/fate_flow/adapter/bfia/wheels/saver.py new file mode 100644 index 000000000..4b63d897e --- /dev/null +++ b/python/fate_flow/adapter/bfia/wheels/saver.py @@ -0,0 +1,30 @@ +from fate_flow.adapter.bfia.utils.entity.status import TaskStatus, JobStatus, EndStatus +from fate_flow.db import ScheduleJob, Task +from fate_flow.entity.types import PROTOCOL +from fate_flow.manager.operation.job_saver import JobSaver, ScheduleJobSaver + + +class BfiaJobSaver(JobSaver): + @classmethod + def check_task_status(cls, old_status, dest_status): + return TaskStatus.StateTransitionRule.if_pass(src_status=old_status, dest_status=dest_status) + + @classmethod + def check_job_status(cls, old_status, dest_status): + return JobStatus.StateTransitionRule.if_pass(src_status=old_status, dest_status=dest_status) + + @classmethod + def end_status_contains(cls, status): + return EndStatus.contains(status) + + @classmethod + def query_task(cls, only_latest=True, reverse=None, order_by=None, protocol=PROTOCOL.BFIA, **kwargs): + return cls._query_task( + Task, only_latest=only_latest, reverse=reverse, order_by=order_by, protocol=protocol, **kwargs + ) + + +class BfiaScheduleJobSaver(ScheduleJobSaver, BfiaJobSaver): + @classmethod + def query_job(cls, reverse=None, order_by=None, protocol=PROTOCOL.BFIA, **kwargs): + return cls._query_job(ScheduleJob, reverse, order_by, protocol=protocol, **kwargs) diff --git a/python/fate_flow/adapter/bfia/wheels/task.py b/python/fate_flow/adapter/bfia/wheels/task.py new file mode 100644 index 000000000..2052b10fb --- /dev/null +++ b/python/fate_flow/adapter/bfia/wheels/task.py @@ -0,0 +1,244 @@ +import json +import os.path +from copy import deepcopy + +from fate_flow.adapter.bfia.settings import VOLUME +from fate_flow.adapter.bfia.utils.entity.status import TaskStatus +from fate_flow.adapter.bfia.utils.spec.job import DagSchemaSpec +from fate_flow.adapter.bfia.wheels.federated import BfiaFederatedScheduler +from fate_flow.adapter.bfia.wheels.parser import get_dag_parser +from fate_flow.adapter.bfia.wheels.saver import BfiaJobSaver as JobSaver +from fate_flow.controller.task import TaskController +from fate_flow.db import Task +from fate_flow.engine.devices.container import ContainerdEngine +from fate_flow.entity.types import PROTOCOL, LauncherType +from fate_flow.manager.service.provider_manager import ProviderManager +from fate_flow.adapter.bfia.settings import LOCAL_SITE_ID as PARTY_ID +from fate_flow.utils import job_utils +from fate_flow.utils.log_utils import schedule_logger + + +class BfiaTaskController(TaskController): + @classmethod + def create_local_tasks(cls, job_id, task_id, task_name, dag): + schedule_logger(job_id).info(f"start create tasks") + dag_schema = DagSchemaSpec.parse_obj(dag) + job_parser = get_dag_parser(dag_schema) + # get runtime roles + roles = job_parser.get_runtime_roles_on_party(task_name, party_id=PARTY_ID) + tasks = [] + for role in roles: + tasks.append(cls.create_local_task(job_id, role, PARTY_ID, task_id, task_name, dag_schema, job_parser)) + schedule_logger(job_id).info("create tasks success") + return tasks + + @classmethod + def create_local_task( + cls, job_id, role, node_id, task_id, task_name, dag_schema, job_parser, task_version=0 + ): + execution_id = job_utils.generate_session_id(task_id, task_version, role, node_id) + task_node = job_parser.get_task_node(task_name=task_name, role=role, party_id=node_id) + task_parser = job_parser.task_parser( + task_node=task_node, job_id=job_id, task_name=task_name, role=role, party_id=node_id, + task_id=task_id, task_version=task_version, parties=job_parser.get_task_runtime_parties(task_name) + ) + task_parameters = task_parser.task_parameters + schedule_logger(job_id).info(f"task {task_name} role {role} part id {node_id} task_parameters" + f" {task_parameters.dict()}, provider: {task_parser.provider}") + task = Task() + task.f_job_id = job_id + task.f_role = role + task.f_party_id = node_id + task.f_task_name = task_name + task.f_component = task_parser.component_ref + task.f_task_id = task_id + task.f_task_version = task_version + task.f_scheduler_party_id = dag_schema.dag.config.initiator.node_id + task.f_status = TaskStatus.READY + task.f_party_status = TaskStatus.READY + task.f_execution_id = execution_id + task.f_provider_name = task_parser.provider + task.f_sync_type = dag_schema.dag.config.job_params.common.sync_type + task.f_task_run = {} + task.f_task_cores = 0 + task.f_protocol = PROTOCOL.BFIA + task.f_component_parameters = task_parameters.dict() + JobSaver.create_task(task.to_human_model_dict()) + return task + + @classmethod + def start_tasks(cls, job_id, task_id, task_name): + # creating a task before starting + jobs = JobSaver.query_job(job_id=job_id) + if not jobs: + raise RuntimeError(f"No found job {job_id}") + job = jobs[0] + tasks = cls.create_local_tasks(job_id, task_id, task_name, job.f_dag) + status_list = [] + + # start + for task in tasks: + schedule_logger(job_id).info(f"start {task.f_role} {task.f_party_id} task") + status_list.append(cls.start_task(task)) + schedule_logger(job_id).info(f"start task status: {status_list}") + if False in status_list: + return False + return True + + @classmethod + def stop_local_task(cls, job_id, task_name): + stop_status = TaskStatus.FAILED + tasks = JobSaver.query_task(job_id=job_id, task_name=task_name) + for task in tasks: + + schedule_logger(job_id=job_id).info(f"[stop]start to kill task {task.f_task_name} " + f"status {task.f_task_status}") + status = cls.stop_task(task, stop_status=stop_status) + schedule_logger(job_id=job_id).info(f"[stop]Kill {task.f_task_name} task completed: {status}") + + @classmethod + def callback_task(cls, task_id, status, role): + BfiaTaskController.update_task_info( + task_info={ + "task_id": task_id, + "party_status": status, + "role": role + }, + callback=BfiaTaskController.update_task_status + ) + + @classmethod + def update_task_info(cls, task_info, callback): + info = deepcopy(task_info) + if "task_version" not in info: + task_info = JobSaver.query_task(task_id=info.get("task_id")) + for task in task_info: + if "role" not in info: + info["role"] = task.f_role + info["party_id"] = task.f_party_id + info["task_version"] = task.f_task_version + info["job_id"] = task.f_job_id + callback(info) + else: + callback(info) + + @classmethod + def update_task(cls, task_info): + update_status = False + try: + update_status = JobSaver.update_task(task_info=task_info) + except Exception as e: + schedule_logger(task_info["job_id"]).exception(e) + finally: + return update_status + + @classmethod + def update_task_status(cls, task_info, scheduler_party_id=None, sync_type=None): + schedule_logger(task_info["job_id"]).info(f"update task status to {task_info.get('party_status')}") + status = task_info.get("status") or task_info.get("party_status") + if status: + task_info["status"] = status.upper() + task_info["party_status"] = status.upper() + update_status = JobSaver.update_task_status(task_info=task_info) + return update_status + + @classmethod + def poll_task(cls, task_id, role): + tasks = JobSaver.query_task(task_id=task_id, party_id=PARTY_ID) + if not tasks: + raise RuntimeError(f"No found task: {task_id} node id {PARTY_ID}") + status = cls.calculate_multi_party_task_status([task.f_party_status for task in tasks]) + return status + + @classmethod + def callback_task_to_scheduler(cls, scheduler_party_id, task_id, status, role): + task_info = { + "task_id": task_id, + "status": status, + "role": role + } + return BfiaFederatedScheduler.request_report_task(party_id=scheduler_party_id, command_body=task_info) + + @classmethod + def query_tasks_status(cls, job_id): + all_status = {} + status = {} + for task in JobSaver.query_task(job_id=job_id): + if task.f_task_id not in all_status: + all_status[task.f_task_id] = [task.f_party_status] + else: + all_status[task.f_task_id].append(task.f_party_status) + + for task_id, status_list in all_status.items(): + status[task_id] = cls.calculate_multi_party_task_status(status_list) + return status + + @classmethod + def calculate_multi_party_task_status(cls, tasks_party_status): + tmp_status_set = set(tasks_party_status) + if len(tmp_status_set) == 1: + return tmp_status_set.pop() + else: + if TaskStatus.FAILED in tmp_status_set: + return TaskStatus.FAILED + if TaskStatus.RUNNING in tmp_status_set: + return TaskStatus.RUNNING + if TaskStatus.READY in tmp_status_set: + return TaskStatus.READY + if TaskStatus.PENDING in tmp_status_set: + return TaskStatus.PENDING + if TaskStatus.SUCCESS in tmp_status_set: + return TaskStatus.SUCCESS + + @classmethod + def build_task_engine(cls, provider_name, launcher_name=LauncherType.DEFAULT): + provider = ProviderManager.get_provider_by_provider_name(provider_name) + return BfiaContainerd(provider) + + @staticmethod + def generate_task_id(): + import uuid + return str(uuid.uuid4()) + + +class BfiaContainerd(ContainerdEngine): + @classmethod + def _get_environment(cls, task: Task, run_parameters): + return cls._flatten_dict(run_parameters) + + @classmethod + def _get_volume(cls, task): + # return { + # os.path.join(LOCAL_LOG_PATH, task.f_job_id, task.f_role, task.f_task_name): + # { + # 'bind': CONTAINER_LOG_PATH, + # 'mode': 'rw' + # } + # } + + return VOLUME + + @classmethod + def _flatten_dict(cls, data, parent_key='', sep='.', loop=True): + special_fields = ["input", "output", "parameter"] + items = {} + for key, value in data.items(): + new_key = f"{parent_key}{sep}{key}" if parent_key else key + # Determine the location of special fields + for field in special_fields: + if new_key.endswith(f"{sep}{field}"): + # continue + items.update(cls._flatten_dict(value, new_key, sep=sep, loop=False)) + break + else: + if isinstance(value, dict) and loop: + items.update(cls._flatten_dict(value, new_key, sep=sep)) + else: + if not loop: + if isinstance(value, dict) or isinstance(value, list): + value = json.dumps(value) + items[new_key] = value + return items + + def exit_with_exception(self, task: Task): + return self.manager.exit_with_exception(self._get_name(task)) diff --git a/python/fate_flow/apps/__init__.py b/python/fate_flow/apps/__init__.py index be45bdb62..d059f1044 100644 --- a/python/fate_flow/apps/__init__.py +++ b/python/fate_flow/apps/__init__.py @@ -13,103 +13,141 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import logging +import os.path import sys +import types +import typing as t + from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path from flask import Blueprint, Flask, request from werkzeug.wrappers.request import Request -from fate_arch.common.base_utils import CustomJSONEncoder - -from fate_flow.entity import RetCode +from fate_flow.controller.permission import PermissionController +from fate_flow.entity.code import ReturnCode from fate_flow.hook import HookManager -from fate_flow.hook.common.parameters import AuthenticationParameters, ClientAuthenticationParameters -from fate_flow.settings import API_VERSION, CLIENT_AUTHENTICATION, SITE_AUTHENTICATION, access_logger -from fate_flow.utils.api_utils import get_json_result, server_error_response +from fate_flow.hook.common.parameters import AuthenticationParameters +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.runtime.system_settings import API_VERSION, CLIENT_AUTHENTICATION, SITE_AUTHENTICATION, \ + ADMIN_PAGE, PARTY_ID +from fate_flow.utils.api_utils import API +from fate_flow.utils.base_utils import CustomJSONEncoder __all__ = ['app'] - -logger = logging.getLogger('flask.app') -for h in access_logger.handlers: - logger.addHandler(h) +app_list = ["client", "partner", "scheduler", "worker"] Request.json = property(lambda self: self.get_json(force=True, silent=True)) app = Flask(__name__) app.url_map.strict_slashes = False -app.json_encoder = CustomJSONEncoder -app.errorhandler(Exception)(server_error_response) +app.errorhandler(422)(API.Output.args_error_response) +app.errorhandler(Exception)(API.Output.server_error_response) +app.json_provider_class = CustomJSONEncoder def search_pages_path(pages_dir): return [path for path in pages_dir.glob('*_app.py') if not path.name.startswith('.')] -def register_page(page_path): - page_name = page_path.stem.rstrip('_app') - module_name = '.'.join(page_path.parts[page_path.parts.index('fate_flow'):-1] + (page_name, )) +def get_app_module(page_path): + page_name = page_path.stem.rstrip('app').rstrip("_") + module_name = '.'.join(page_path.parts[page_path.parts.index('fate_flow')+2:-1] + (page_name, )) + return module_name + +def register_page(page_path, func=None, prefix=API_VERSION): + page_name = page_path.stem.rstrip('app').rstrip("_") + module_name = '.'.join(page_path.parts[page_path.parts.index('fate_flow')+2:-1] + (page_name, )) spec = spec_from_file_location(module_name, page_path) page = module_from_spec(spec) page.app = app page.manager = Blueprint(page_name, module_name) + rule_methods_list = [] + + # rewrite blueprint route to get rule_list + def route(self, rule: str, **options: t.Any) -> t.Callable: + def decorator(f: t.Callable) -> t.Callable: + endpoint = options.pop("endpoint", None) + rule_methods_list.append((rule, options.get("methods", []))) + self.add_url_rule(rule, endpoint, f, **options) + return f + return decorator + + page.manager.route = types.MethodType(route, page.manager) + + if func: + page.manager.before_request(func) sys.modules[module_name] = page spec.loader.exec_module(page) - page_name = getattr(page, 'page_name', page_name) - url_prefix = f'/{API_VERSION}/{page_name}' - + url_prefix = f'/{prefix}/{page_name}' app.register_blueprint(page.manager, url_prefix=url_prefix) - return url_prefix - - -client_urls_prefix = [ - register_page(path) - for path in search_pages_path(Path(__file__).parent) -] -scheduling_urls_prefix = [ - register_page(path) - for path in search_pages_path(Path(__file__).parent.parent / 'scheduling_apps') -] + return page_name, [(os.path.join(url_prefix, rule_methods[0].lstrip("/")), rule_methods[1]) for rule_methods in rule_methods_list] def client_authentication_before_request(): - for url_prefix in scheduling_urls_prefix: - if request.path.startswith(url_prefix): - return - - result = HookManager.client_authentication(ClientAuthenticationParameters( - request.full_path, request.headers, - request.form, request.data, request.json, - )) + if CLIENT_AUTHENTICATION: + result = HookManager.client_authentication(AuthenticationParameters( + request.path, request.method, request.headers, + request.form, request.data, request.json, request.full_path + )) - if result.code != RetCode.SUCCESS: - return get_json_result(result.code, result.message) + if result.code != ReturnCode.Base.SUCCESS: + return API.Output.json(result.code, result.message) def site_authentication_before_request(): - for url_prefix in client_urls_prefix: - if request.path.startswith(url_prefix): - return - - result = HookManager.site_authentication(AuthenticationParameters( - request.headers.get('src_party_id'), - request.headers.get('site_signature'), - request.json, - )) - - if result.code != RetCode.SUCCESS: - return get_json_result(result.code, result.message) - - -@app.before_request -def authentication_before_request(): - if CLIENT_AUTHENTICATION: - return client_authentication_before_request() - if SITE_AUTHENTICATION: - return site_authentication_before_request() + result = HookManager.site_authentication(AuthenticationParameters( + request.path, request.method, request.headers, + request.form, request.data, request.json, request.full_path + )) + + if result.code != ReturnCode.Base.SUCCESS: + return API.Output.json(result.code, result.message) + + +def init_apps(): + urls_dict = {} + before_request_func = { + "client": client_authentication_before_request, + "partner": site_authentication_before_request, + "scheduler": site_authentication_before_request + } + for key in app_list: + urls_dict[key] = [register_page(path, before_request_func.get(key)) for path in search_pages_path(Path(__file__).parent / key)] + # adapter extend apps + try: + from fate_flow.adapter import load_adapter_apps + urls_dict.update(load_adapter_apps(register_page, search_pages_path)) + except: + pass + if CLIENT_AUTHENTICATION or SITE_AUTHENTICATION: + _init_permission_group(urls=urls_dict) + + +def _init_permission_group(urls: dict): + for role, role_items in urls.items(): + super_role = "super_" + role + if role in ["scheduler", "partner"]: + role = "site" + super_role = "site" + RuntimeConfig.set_client_roles(role, super_role) + for resource, rule_methods_list in role_items: + for rule_methods in rule_methods_list: + rule = rule_methods[0] + methods = rule_methods[1] + for method in methods: + if resource in ADMIN_PAGE: + PermissionController.add_policy(super_role, rule, method) + else: + PermissionController.add_policy(super_role, rule, method) + PermissionController.add_policy(role, rule, method) + PermissionController.add_role_for_user("admin", super_role, init=True) + PermissionController.add_role_for_user(PARTY_ID, "site", init=True) + + +init_apps() diff --git a/python/fate_flow/apps/checkpoint_app.py b/python/fate_flow/apps/checkpoint_app.py deleted file mode 100644 index 6b37eae05..000000000 --- a/python/fate_flow/apps/checkpoint_app.py +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from flask import request, abort - -from fate_flow.model.checkpoint import CheckpointManager -from fate_flow.utils.api_utils import error_response, get_json_result -from fate_flow.utils.detect_utils import check_config - - -def load_checkpoints(): - required_args = ['role', 'party_id', 'model_id', 'model_version', 'component_name'] - try: - check_config(request.json, required_args) - except Exception as e: - abort(error_response(400, str(e))) - - checkpoint_manager = CheckpointManager(**{i: request.json[i] for i in required_args}) - checkpoint_manager.load_checkpoints_from_disk() - return checkpoint_manager - - -@manager.route('/list', methods=['POST']) -def list_checkpoints(): - checkpoint_manager = load_checkpoints() - return get_json_result(data=checkpoint_manager.to_dict()) - - -@manager.route('/get', methods=['POST']) -def get_checkpoint(): - checkpoint_manager = load_checkpoints() - - if 'step_index' in request.json: - try: - request.json['step_index'] = int(request.json['step_index']) - except Exception: - return error_response(400, "Invalid 'step_index'") - - checkpoint = checkpoint_manager.get_checkpoint_by_index(request.json['step_index']) - elif 'step_name' in request.json: - checkpoint = checkpoint_manager.get_checkpoint_by_name(request.json['step_name']) - else: - return error_response(400, "'step_index' or 'step_name' is required") - - if checkpoint is None: - return error_response(404, "The checkpoint was not found.") - - return get_json_result(data=checkpoint.to_dict(True)) diff --git a/python/fate_flow/apps/client/__init__.py b/python/fate_flow/apps/client/__init__.py new file mode 100644 index 000000000..ee85ecebf --- /dev/null +++ b/python/fate_flow/apps/client/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/python/fate_flow/apps/client/client_app.py b/python/fate_flow/apps/client/client_app.py new file mode 100644 index 000000000..18e724e2e --- /dev/null +++ b/python/fate_flow/apps/client/client_app.py @@ -0,0 +1,91 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from webargs import fields + +from fate_flow.apps.desc import APP_NAME, APP_ID, PARTY_ID, SITE_APP_ID, SITE_APP_TOKEN +from fate_flow.entity.code import ReturnCode +from fate_flow.entity.types import AppType +from fate_flow.manager.service.app_manager import AppManager +from fate_flow.runtime.system_settings import APP_MANAGER_PAGE +from fate_flow.utils.api_utils import API + +page_name = APP_MANAGER_PAGE + + +@manager.route('/client/create', methods=['POST']) +@API.Input.json(app_name=fields.String(required=True), desc=APP_NAME) +def create_client_app(app_name): + data = AppManager.create_app(app_name=app_name, app_type=AppType.CLIENT, init=False) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=data) + + +@manager.route('/client/delete', methods=['POST']) +@API.Input.json(app_id=fields.String(required=True), desc=APP_ID) +def delete_client_app(app_id): + status = AppManager.delete_app(app_id=app_id, app_type=AppType.CLIENT, init=False) + return API.Output.json(data={"status": status}) + + +@manager.route('/client/query', methods=['GET']) +@API.Input.params(app_id=fields.String(required=False), desc=APP_ID) +@API.Input.params(app_name=fields.String(required=False), desc=APP_NAME) +def query_client_app(app_id=None, app_name=None): + apps = AppManager.query_app(app_id=app_id, app_name=app_name) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=[app.to_human_model_dict() for app in apps]) + + +@manager.route('/site/create', methods=['POST']) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +def create_site_app(party_id): + data = AppManager.create_app(app_name=party_id, app_type=AppType.SITE) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=data) + + +@manager.route('/site/delete', methods=['POST']) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +def delete_site_app(party_id): + status = AppManager.delete_app(app_name=party_id, app_type=AppType.SITE, init=True) + return API.Output.json(data={"status": status}) + + +@manager.route('/site/query', methods=['GET']) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +def query_site_app(party_id=None): + apps = AppManager.query_app(app_name=party_id, app_type=AppType.SITE,init=True) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=[app.to_human_model_dict() for app in apps]) + + +@manager.route('/partner/create', methods=['POST']) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.json(app_id=fields.String(required=True), desc=SITE_APP_ID) +@API.Input.json(app_token=fields.String(required=True), desc=SITE_APP_TOKEN) +def create_partner_app(party_id, app_id, app_token): + data = AppManager.create_partner_app(app_id=app_id, party_id=party_id, app_token=app_token) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=data) + + +@manager.route('/partner/delete', methods=['POST']) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +def delete_partner_app(party_id): + status = AppManager.delete_partner_app(party_id=party_id, init=False) + return API.Output.json(data={"status": status}) + + +@manager.route('/partner/query', methods=['GET']) +@API.Input.params(party_id=fields.String(required=False), desc=PARTY_ID) +def query_partner_app(party_id=None): + apps = AppManager.query_partner_app(party_id=party_id) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=[app.to_human_model_dict() for app in apps]) diff --git a/python/fate_flow/apps/client/data_app.py b/python/fate_flow/apps/client/data_app.py new file mode 100644 index 000000000..2612b63f0 --- /dev/null +++ b/python/fate_flow/apps/client/data_app.py @@ -0,0 +1,106 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +from webargs import fields +from flask import request +from fate_flow.apps.desc import SERVER_FILE_PATH, HEAD, PARTITIONS, META, EXTEND_SID, NAMESPACE, NAME, DATA_WAREHOUSE, \ + DROP, SITE_NAME +from fate_flow.engine import storage +from fate_flow.manager.components.component_manager import ComponentManager +from fate_flow.manager.outputs.data import DataManager +from fate_flow.utils.api_utils import API +from fate_flow.errors.server_error import NoFoundTable, NoFoundFile + +page_name = "data" + + +@manager.route('/component/upload', methods=['POST']) +@API.Input.json(file=fields.String(required=True), desc=SERVER_FILE_PATH) +@API.Input.json(head=fields.Bool(required=True), desc=HEAD) +@API.Input.json(partitions=fields.Integer(required=True), desc=PARTITIONS) +@API.Input.json(meta=fields.Dict(required=True), desc=META) +@API.Input.json(extend_sid=fields.Bool(required=False), desc=EXTEND_SID) +@API.Input.json(namespace=fields.String(required=False), desc=NAMESPACE) +@API.Input.json(name=fields.String(required=False), desc=NAME) +def upload_data(file, head, partitions, meta, namespace=None, name=None, extend_sid=False): + if namespace and name: + result = ComponentManager.upload_dataframe( + file=file, head=head, partitions=partitions, meta=meta, namespace=namespace, name=name, extend_sid=extend_sid + ) + else: + result = ComponentManager.upload( + file=file, head=head, partitions=partitions, meta=meta, namespace=namespace, name=name, + extend_sid=extend_sid + ) + return API.Output.json(**result) + + +@manager.route('/component/upload/file', methods=['POST']) +@API.Input.form(head=fields.Bool(required=True), desc=HEAD) +@API.Input.form(partitions=fields.Integer(required=True), desc=PARTITIONS) +@API.Input.form(meta=fields.String(required=True), desc=META) +@API.Input.form(extend_sid=fields.Bool(required=False), desc=EXTEND_SID) +@API.Input.form(namespace=fields.String(required=False), desc=NAMESPACE) +@API.Input.form(name=fields.String(required=False), desc=NAME) +def upload_file(head, partitions, meta, namespace=None, name=None, extend_sid=False): + + file = request.files.get('file') + if not file: + raise NoFoundFile() + result = ComponentManager.upload_file( + file=file, head=head, partitions=partitions, meta=json.loads(meta), namespace=namespace, name=name, + extend_sid=extend_sid + ) + + return API.Output.json(**result) + + +@manager.route('/component/download', methods=['POST']) +@API.Input.json(name=fields.String(required=True), desc=NAME) +@API.Input.json(namespace=fields.String(required=True), desc=NAMESPACE) +@API.Input.json(path=fields.String(required=False), desc=SERVER_FILE_PATH) +def download_data(namespace, name, path): + result = ComponentManager.download( + path=path, namespace=namespace, name=name + ) + return API.Output.json(**result) + + +@manager.route('/component/dataframe/transformer', methods=['POST']) +@API.Input.json(data_warehouse=fields.Dict(required=True), desc=DATA_WAREHOUSE) +@API.Input.json(namespace=fields.String(required=True), desc=NAMESPACE) +@API.Input.json(name=fields.String(required=True), desc=NAME) +@API.Input.json(site_name=fields.String(required=False), desc=SITE_NAME) +@API.Input.json(drop=fields.Bool(required=False), desc=DROP) +def transformer_data(data_warehouse, namespace, name, drop=True, site_name=None): + result = ComponentManager.dataframe_transformer(data_warehouse, namespace, name, drop, site_name) + return API.Output.json(**result) + + +@manager.route('/download', methods=['GET']) +@API.Input.params(name=fields.String(required=True), desc=NAME) +@API.Input.params(namespace=fields.String(required=True), desc=NAMESPACE) +@API.Input.params(header=fields.String(required=False), desc=HEAD) +def download(namespace, name, header=None): + data_table_meta = storage.StorageTableMeta(name=name, namespace=namespace) + if not data_table_meta: + raise NoFoundTable(name=name, namespace=namespace) + return DataManager.send_table( + output_tables_meta={"data": data_table_meta}, + tar_file_name=f'download_data_{namespace}_{name}.tar.gz', + need_head=header + ) + diff --git a/python/fate_flow/apps/client/job_app.py b/python/fate_flow/apps/client/job_app.py new file mode 100644 index 000000000..cab3b138e --- /dev/null +++ b/python/fate_flow/apps/client/job_app.py @@ -0,0 +1,217 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import io +import os +import tarfile + +from webargs import fields + +from fate_flow.apps.desc import DAG_SCHEMA, USER_NAME, JOB_ID, ROLE, PARTY_ID, STATUS, LIMIT, PAGE, PARTNER, ORDER_BY, \ + ORDER, DESCRIPTION, TASK_NAME, TASK_ID, TASK_VERSION, NODES +from fate_flow.controller.job import JobController +from fate_flow.entity.code import ReturnCode +from fate_flow.entity.spec.dag import DAGSchema +from fate_flow.entity.spec.flow import SubmitJobInput, QueryJobInput, StopJobInput, QueryTaskInput +from fate_flow.entity.types import PROTOCOL +from fate_flow.errors.server_error import NoFoundJob, NoFoundTask, FileNoFound +from fate_flow.manager.operation.job_saver import JobSaver +from fate_flow.utils import job_utils +from fate_flow.utils.api_utils import API +from fate_flow.manager.pipeline import pipeline as pipeline_manager + + +@manager.route('/submit', methods=['POST']) +@API.Input.json(dag_schema=fields.Dict(required=True), desc=DAG_SCHEMA) +@API.Input.headers(user_name=fields.String(required=False), desc=USER_NAME) +def submit_job(dag_schema, user_name=None): + dag_schema = DAGSchema(**dag_schema) + if dag_schema.kind == PROTOCOL.FATE_FLOW: + submit_result = JobController.request_create_job(dag_schema, user_name) + else: + from fate_flow.adapter import AdapterJobController + submit_result = AdapterJobController(dag_schema.kind).create_job(SubmitJobInput(dag_schema=dag_schema)).dict() + return API.Output.json(**submit_result) + + +@manager.route('/query', methods=['GET']) +@API.Input.params(job_id=fields.String(required=False), desc=JOB_ID) +@API.Input.params(role=fields.String(required=False), desc=ROLE) +@API.Input.params(party_id=fields.String(required=False), desc=PARTY_ID) +@API.Input.params(status=fields.String(required=False), desc=STATUS) +@API.Input.headers(user_name=fields.String(required=False), desc=USER_NAME) +def query_job(job_id=None, role=None, party_id=None, status=None, user_name=None): + jobs = JobController.query_job(job_id=job_id, role=role, party_id=party_id, status=status, user_name=user_name) + if not jobs: + return API.Output.fate_flow_exception(NoFoundJob(job_id=job_id, role=role, party_id=party_id, status=status)) + kind = jobs[0].f_protocol + + if kind != PROTOCOL.FATE_FLOW: + from fate_flow.adapter import AdapterJobController + jobs = AdapterJobController(kind).query_job(QueryJobInput(jobs=jobs)).jobs + + return API.Output.json(data=[job.to_human_model_dict() for job in jobs]) + + +@manager.route('/stop', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True), desc=JOB_ID) +def request_stop_job(job_id=None): + jobs = JobSaver.query_job(job_id=job_id) + if not jobs: + raise NoFoundJob(job_id=job_id) + kind = jobs[0].f_protocol + + if kind != PROTOCOL.FATE_FLOW: + from fate_flow.adapter import AdapterJobController + stop_result = AdapterJobController(kind).stop_job(StopJobInput(job_id=job_id)).dict() + + else: + stop_result = JobController.request_stop_job(job_id, jobs=jobs) + return API.Output.json(**stop_result) + + +@manager.route('/rerun', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True), desc=JOB_ID) +def request_rerun_job(job_id=None): + jobs = JobController.query_job(job_id=job_id) + if not jobs: + return API.Output.fate_flow_exception(NoFoundJob(job_id=job_id)) + rerun_result = JobController.request_rerun_job(job=jobs[0]) + return API.Output.json(**rerun_result) + + +@manager.route('/list/query', methods=['GET']) +@API.Input.params(limit=fields.Integer(required=False), desc=LIMIT) +@API.Input.params(page=fields.Integer(required=False), desc=PAGE) +@API.Input.params(job_id=fields.String(required=False), desc=JOB_ID) +@API.Input.params(description=fields.String(required=False), desc=DESCRIPTION) +@API.Input.params(partner=fields.String(required=False), desc=PARTNER) +@API.Input.params(party_id=fields.String(required=False), desc=PARTY_ID) +@API.Input.params(role=fields.List(fields.Str(), required=False), desc=ROLE) +@API.Input.params(status=fields.List(fields.Str(), required=False), desc=STATUS) +@API.Input.params(order_by=fields.String(required=False), desc=ORDER_BY) +@API.Input.params(order=fields.String(required=False), desc=ORDER) +@API.Input.headers(user_name=fields.String(required=False), desc=USER_NAME) +def query_job_list(limit=0, page=0, job_id=None, description=None, partner=None, party_id=None, role=None, status=None, + order_by=None, order=None, user_name=None): + count, data = JobController.query_job_list( + limit, page, job_id, description, partner, party_id, role, status, order_by, order, user_name + ) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", + data={"count": count, "data": data}) + + +@manager.route('/task/query', methods=['GET']) +@API.Input.params(job_id=fields.String(required=False), desc=JOB_ID) +@API.Input.params(role=fields.String(required=False),desc=ROLE) +@API.Input.params(party_id=fields.String(required=False), desc=PARTY_ID) +@API.Input.params(status=fields.String(required=False), desc=STATUS) +@API.Input.params(task_name=fields.String(required=False), desc=TASK_NAME) +@API.Input.params(task_id=fields.String(required=False), desc=TASK_ID) +@API.Input.params(task_version=fields.Integer(required=False), desc=TASK_VERSION) +def query_task(job_id=None, role=None, party_id=None, status=None, task_name=None, task_id=None, task_version=None): + tasks = JobController.query_tasks(job_id=job_id, role=role, party_id=party_id, status=status, task_name=task_name, + task_id=task_id, task_version=task_version, ignore_protocol=True) + if not tasks: + return API.Output.fate_flow_exception(NoFoundTask()) + + kind = tasks[0].f_protocol + if kind != PROTOCOL.FATE_FLOW: + from fate_flow.adapter import AdapterJobController + tasks = AdapterJobController(kind).query_task(QueryTaskInput(tasks=tasks)).tasks + + return API.Output.json(data=[task.to_human_model_dict() for task in tasks]) + + +@manager.route('/task/list/query', methods=['GET']) +@API.Input.params(limit=fields.Integer(required=False), desc=LIMIT) +@API.Input.params(page=fields.Integer(required=False), desc=PAGE) +@API.Input.params(job_id=fields.String(required=False), desc=JOB_ID) +@API.Input.params(role=fields.String(required=False), desc=ROLE) +@API.Input.params(party_id=fields.String(required=False), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=False), desc=TASK_NAME) +@API.Input.params(order_by=fields.String(required=False), desc=ORDER_BY) +@API.Input.params(order=fields.String(required=False), desc=ORDER) +def query_task_list(limit=0, page=0, job_id=None, role=None, party_id=None, task_name=None, order_by=None, order=None): + count, data = JobController.query_task_list( + limit, page, job_id, role, party_id, task_name, order_by, order + ) + return API.Output.json( + code=ReturnCode.Base.SUCCESS, message="success", + data={"count": count, "data": data} + ) + + +@manager.route('/log/download', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True), desc=JOB_ID) +def download_job_logs(job_id): + job_log_dir = job_utils.get_job_log_directory(job_id=job_id) + if not os.path.exists(job_log_dir): + return API.Output.fate_flow_exception(e=FileNoFound(path=job_log_dir)) + memory_file = io.BytesIO() + with tarfile.open(fileobj=memory_file, mode='w:gz') as tar: + for root, _, files in os.walk(job_log_dir): + for file in files: + full_path = os.path.join(root, file) + rel_path = os.path.relpath(full_path, job_log_dir) + tar.add(full_path, rel_path) + memory_file.seek(0) + return API.Output.file( + memory_file, attachment_filename=f'job_{job_id}_log.tar.gz', as_attachment=True, mimetype="application/gzip" + ) + + +@manager.route('/queue/clean', methods=['POST']) +def clean_queue(): + data = JobController.clean_queue() + return API.Output.json(data=data) + + +@manager.route('/clean', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True), desc=JOB_ID) +def clean_job(job_id): + JobController.clean_job(job_id=job_id) + return API.Output.json() + + +@manager.route('/data/view', methods=['GET']) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +def data_view(job_id, role, party_id): + data = JobController.data_view(job_id, role, party_id) + return API.Output.json(data=data) + + +@manager.route('/notes/add', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.json(role=fields.String(required=True), desc=ROLE) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.json(notes=fields.String(required=True), desc=NODES) +def add_notes(job_id, role, party_id, notes): + JobController.add_notes(job_id=job_id, role=role, party_id=party_id, notes=notes) + return API.Output.json() + + +@manager.route('/dag/dependency', methods=['GET']) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +def dag_dependency(job_id, role, party_id): + jobs = JobController.query_job(job_id=job_id, role=role, party_id=party_id) + if not jobs: + return API.Output.fate_flow_exception(NoFoundJob(job_id=job_id, role=role, party_id=party_id)) + data = pipeline_manager.pipeline_dag_dependency(jobs[0]) + return API.Output.json(data=data) diff --git a/python/fate_flow/apps/client/log_app.py b/python/fate_flow/apps/client/log_app.py new file mode 100644 index 000000000..3d6ec2acc --- /dev/null +++ b/python/fate_flow/apps/client/log_app.py @@ -0,0 +1,49 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from webargs import fields + +from fate_flow.apps.desc import LOG_TYPE, JOB_ID, ROLE, PARTY_ID, TASK_NAME, INSTANCE_ID, BEGIN, END +from fate_flow.manager.outputs.log import LogManager +from fate_flow.utils.api_utils import API +from fate_flow.utils.wraps_utils import cluster_route + + +@manager.route('/count', methods=['GET']) +@API.Input.params(log_type=fields.String(required=True), desc=LOG_TYPE) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=False), desc=ROLE) +@API.Input.params(party_id=fields.String(required=False), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=False), desc=TASK_NAME) +@API.Input.params(instance_id=fields.String(required=False), desc=INSTANCE_ID) +@cluster_route +def count(log_type, job_id, role=None, party_id=None, task_name=None, instance_id=None): + data = LogManager(log_type, job_id, role=role, party_id=party_id, task_name=task_name).count() + return API.Output.json(data=data) + + +@manager.route('/query', methods=['GET']) +@API.Input.params(log_type=fields.String(required=True), desc=LOG_TYPE) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=False), desc=TASK_NAME) +@API.Input.params(begin=fields.Integer(required=False), desc=BEGIN) +@API.Input.params(end=fields.Integer(required=False), desc=END) +@API.Input.params(instance_id=fields.String(required=False), desc=INSTANCE_ID) +@cluster_route +def get(log_type, job_id, role, party_id, task_name=None, begin=None, end=None, instance_id=None): + data = LogManager(log_type, job_id, role=role, party_id=party_id, task_name=task_name).cat_log(begin=begin, end=end) + return API.Output.json(data=data) diff --git a/python/fate_flow/apps/client/model_app.py b/python/fate_flow/apps/client/model_app.py new file mode 100644 index 000000000..b8e4e544d --- /dev/null +++ b/python/fate_flow/apps/client/model_app.py @@ -0,0 +1,99 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os.path +from tempfile import TemporaryDirectory + +from flask import request +from webargs import fields + +from fate_flow.apps.desc import MODEL_ID, MODEL_VERSION, PARTY_ID, ROLE, SERVER_DIR_PATH, TASK_NAME, OUTPUT_KEY +from fate_flow.errors.server_error import NoFoundFile +from fate_flow.manager.outputs.model import PipelinedModel +from fate_flow.utils.api_utils import API + + +@manager.route('/load', methods=['POST']) +def load(): + # todo: + return API.Output.json() + + +@manager.route('/migrate', methods=['POST']) +def migrate(): + # todo: + return API.Output.json() + + +@manager.route('/export', methods=['POST']) +@API.Input.json(model_id=fields.String(required=True), desc=MODEL_ID) +@API.Input.json(model_version=fields.String(required=True), desc=MODEL_VERSION) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.json(role=fields.String(required=True), desc=ROLE) +@API.Input.json(path=fields.String(required=True), desc=SERVER_DIR_PATH) +def export(model_id, model_version, party_id, role, path): + file_list = PipelinedModel.export_model( + model_id=model_id, + model_version=model_version, + party_id=party_id, + role=role, + dir_path=path + ) + return API.Output.json(data=file_list) + + +@manager.route('/import', methods=['POST']) +@API.Input.form(model_id=fields.String(required=True), desc=MODEL_ID) +@API.Input.form(model_version=fields.String(required=True), desc=MODEL_VERSION) +def import_model(model_id, model_version): + file = request.files.get('file') + if not file: + raise NoFoundFile() + with TemporaryDirectory() as temp_dir: + path = os.path.join(temp_dir, file.name) + file.save(path) + PipelinedModel.import_model(model_id, model_version, path, temp_dir) + return API.Output.json() + + +@manager.route('/delete', methods=['POST']) +@API.Input.json(model_id=fields.String(required=True), desc=MODEL_ID) +@API.Input.json(model_version=fields.String(required=True), desc=MODEL_VERSION) +@API.Input.json(role=fields.String(required=False), desc=ROLE) +@API.Input.json(party_id=fields.String(required=False), desc=PARTY_ID) +@API.Input.json(task_name=fields.String(required=False), desc=TASK_NAME) +@API.Input.json(output_key=fields.String(required=False), desc=OUTPUT_KEY) +def delete_model(model_id, model_version, role=None, party_id=None, task_name=None, output_key=None): + count = PipelinedModel.delete_model( + model_id=model_id, + model_version=model_version, + party_id=party_id, + role=role, + task_name=task_name, + output_key=output_key + ) + return API.Output.json(data={"count": count}) + + +@manager.route('/store', methods=['POST']) +def store(): + # todo: + return API.Output.json() + + +@manager.route('/restore', methods=['POST']) +def restore(): + # todo: + return API.Output.json() diff --git a/python/fate_flow/apps/client/output_app.py b/python/fate_flow/apps/client/output_app.py new file mode 100644 index 000000000..5e74a751f --- /dev/null +++ b/python/fate_flow/apps/client/output_app.py @@ -0,0 +1,209 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from webargs import fields + +from fate_flow.apps.desc import JOB_ID, ROLE, PARTY_ID, TASK_NAME, FILTERS, OUTPUT_KEY +from fate_flow.entity.code import ReturnCode +from fate_flow.entity.types import PROTOCOL +from fate_flow.errors.server_error import NoFoundTask +from fate_flow.manager.outputs.data import DataManager +from fate_flow.manager.outputs.model import PipelinedModel +from fate_flow.manager.outputs.metric import OutputMetric +from fate_flow.manager.operation.job_saver import JobSaver +from fate_flow.utils.api_utils import API + + +@manager.route('/metric/key/query', methods=['GET']) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=True), desc=TASK_NAME) +def query_metric_key(job_id, role, party_id, task_name): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + metric_keys = OutputMetric(job_id=job_id, role=role, party_id=party_id, task_name=task_name, + task_id=tasks[0].f_task_id, task_version=tasks[0].f_task_version).query_metric_keys() + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success', data=metric_keys) + + +@manager.route('/metric/query', methods=['GET']) +@API.Input.params(job_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=True), desc=TASK_NAME) +@API.Input.params(filters=fields.Dict(required=False), desc=FILTERS) +def query_metric(job_id, role, party_id, task_name, filters=None): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name, ignore_protocol=True) + if not tasks: + return API.Output.fate_flow_exception( + e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + ) + + kind = tasks[0].f_protocol + if kind != PROTOCOL.FATE_FLOW: + from fate_flow.adapter import AdapterJobController + metrics = AdapterJobController(kind).query_output_metric() + else: + metrics = OutputMetric( + job_id=job_id, + role=role, + party_id=party_id, + task_name=task_name, + task_id=tasks[0].f_task_id, + task_version=tasks[0].f_task_version + ).read_metrics(filters) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success', data=metrics) + + +@manager.route('/metric/delete', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.json(role=fields.String(required=True), desc=ROLE) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.json(task_name=fields.String(required=True), desc=TASK_NAME) +def delete_metric(job_id, role, party_id, task_name): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + metric_keys = OutputMetric( + job_id=job_id, role=role, party_id=party_id, task_name=task_name, + task_id=tasks[0].f_task_id, task_version=tasks[0].f_task_version + ).delete_metrics() + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success', data=metric_keys) + + +@manager.route('/model/query', methods=['GET']) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=True), desc=TASK_NAME) +def query_model(job_id, role, party_id, task_name): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name, ignore_protocol=True) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + task = tasks[0] + + kind = task.f_protocol + if kind != PROTOCOL.FATE_FLOW: + from fate_flow.adapter import AdapterJobController + model_data = AdapterJobController(kind).query_output_model() + else: + model_data = PipelinedModel.read_model(task.f_job_id, task.f_role, task.f_party_id, task.f_task_name) + return API.Output.json(data=model_data) + + +@manager.route('/model/download', methods=['GET']) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=True), desc=TASK_NAME) +def download(job_id, role, party_id, task_name): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + task = tasks[0] + return PipelinedModel.download_model(job_id=task.f_job_id, role=task.f_role, party_id=task.f_party_id, + task_name=task.f_task_name) + + +@manager.route('/model/delete', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.json(role=fields.String(required=True), desc=ROLE) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.json(task_name=fields.String(required=True), desc=TASK_NAME) +def delete_model(job_id, role, party_id, task_name): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + task = tasks[0] + PipelinedModel.delete_model( + job_id=task.f_job_id, + role=task.f_role, + party_id=task.f_party_id, + task_name=task.f_task_name) + return API.Output.json() + + +@manager.route('/data/download', methods=['GET']) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=True), desc=TASK_NAME) +@API.Input.params(output_key=fields.String(required=False), desc=OUTPUT_KEY) +def output_data_download(job_id, role, party_id, task_name, output_key=None): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + task = tasks[0] + return DataManager.download_output_data( + job_id=task.f_job_id, + role=task.f_role, + party_id=task.f_party_id, + task_name=task.f_task_name, + task_id=task.f_task_id, + task_version=task.f_task_version, + output_key=output_key, + tar_file_name=f"{job_id}_{role}_{party_id}_{task_name}" + + ) + + +@manager.route('/data/table', methods=['GET']) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=True), desc=TASK_NAME) +def output_data_table(job_id, role, party_id, task_name): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + task = tasks[0] + return DataManager.query_output_data_table( + job_id=task.f_job_id, + role=task.f_role, + party_id=task.f_party_id, + task_name=task.f_task_name, + task_id=task.f_task_id, + task_version=task.f_task_version + ) + + +@manager.route('/data/display', methods=['GET']) +@API.Input.params(job_id=fields.String(required=True), desc=JOB_ID) +@API.Input.params(role=fields.String(required=True), desc=ROLE) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(task_name=fields.String(required=True), desc=TASK_NAME) +def output_data_display(job_id, role, party_id, task_name): + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + task = tasks[0] + return DataManager.display_output_data( + job_id=task.f_job_id, + role=task.f_role, + party_id=task.f_party_id, + task_name=task.f_task_name, + task_id=task.f_task_id, + task_version=task.f_task_version + ) diff --git a/python/fate_flow/apps/client/permission_app.py b/python/fate_flow/apps/client/permission_app.py new file mode 100644 index 000000000..62f84754b --- /dev/null +++ b/python/fate_flow/apps/client/permission_app.py @@ -0,0 +1,88 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from webargs import fields + +from fate_flow.apps.desc import PERMISSION_APP_ID, PERMISSION_ROLE, PARTY_ID, COMPONENT, DATASET +from fate_flow.controller.permission import ResourcePermissionController, PermissionController +from fate_flow.entity.code import ReturnCode +from fate_flow.entity.types import PermissionParameters +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.runtime.system_settings import PERMISSION_MANAGER_PAGE +from fate_flow.utils.api_utils import API + +page_name = PERMISSION_MANAGER_PAGE + + +@manager.route('/grant', methods=['POST']) +@API.Input.json(app_id=fields.String(required=True), desc=PERMISSION_APP_ID) +@API.Input.json(role=fields.String(required=True), desc=PERMISSION_ROLE) +def grant(app_id, role): + for roles in PermissionController.get_roles_for_user(app_id=app_id): + PermissionController.delete_role_for_user(app_id=app_id, role=roles, grant_role=role) + status = PermissionController.add_role_for_user(app_id=app_id, role=role) + return API.Output.json(data={"status": status}) + + +@manager.route('/delete', methods=['POST']) +@API.Input.json(app_id=fields.String(required=True), desc=PERMISSION_APP_ID) +@API.Input.json(role=fields.String(required=True), desc=PERMISSION_ROLE) +def delete(app_id, role): + status = PermissionController.delete_role_for_user(app_id=app_id, role=role) + return API.Output.json(data={"status": status}) + + +@manager.route('/query', methods=['GET']) +@API.Input.params(app_id=fields.String(required=True), desc=PERMISSION_APP_ID) +def query(app_id): + permissions = {} + for role in PermissionController.get_roles_for_user(app_id=app_id): + permissions[role] = PermissionController.get_permissions_for_user(app_id=role) + return API.Output.json(code=ReturnCode.Base.SUCCESS, data=permissions) + + +@manager.route('/role/query', methods=['GET']) +def query_roles(): + return API.Output.json(data=RuntimeConfig.CLIENT_ROLE) + + +@manager.route('/resource/grant', methods=['post']) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.json(component=fields.String(required=False), desc=COMPONENT) +@API.Input.json(dataset=fields.List(fields.Dict(), required=False), desc=DATASET) +def grant_resource_permission(party_id, component=None, dataset=None): + parameters = PermissionParameters(party_id=party_id, component=component, dataset=dataset) + ResourcePermissionController(party_id).grant_or_delete(parameters) + return API.Output.json() + + +@manager.route('/resource/delete', methods=['post']) +@API.Input.json(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.json(component=fields.String(required=False), desc=COMPONENT) +@API.Input.json(dataset=fields.List(fields.Dict(), required=False), desc=DATASET) +def delete_resource_permission(party_id, component=None, dataset=None): + parameters = PermissionParameters(party_id=party_id, component=component, dataset=dataset, is_delete=True) + ResourcePermissionController(parameters.party_id).grant_or_delete(parameters) + return API.Output.json() + + +@manager.route('/resource/query', methods=['get']) +@API.Input.params(party_id=fields.String(required=True), desc=PARTY_ID) +@API.Input.params(component=fields.String(required=False), desc=COMPONENT) +@API.Input.params(dataset=fields.Dict(required=False), desc=DATASET) +def query_resource_privilege(party_id, component=None, dataset=None): + parameters = PermissionParameters(party_id=party_id, component=component, dataset=dataset) + data = ResourcePermissionController(parameters.party_id).query() + return API.Output.json(data=data) diff --git a/python/fate_flow/apps/client/provider_app.py b/python/fate_flow/apps/client/provider_app.py new file mode 100644 index 000000000..c5f7dd962 --- /dev/null +++ b/python/fate_flow/apps/client/provider_app.py @@ -0,0 +1,58 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from webargs import fields + +from fate_flow.apps.desc import PROVIDER_NAME, DEVICE, VERSION, COMPONENT_METADATA, PROVIDER_ALL_NAME, \ + COMPONENTS_DESCRIPTION, PROTOCOL +from fate_flow.errors.server_error import DeviceNotSupported +from fate_flow.manager.service.provider_manager import ProviderManager +from fate_flow.utils.api_utils import API + + +@manager.route('/register', methods=['POST']) +@API.Input.json(name=fields.String(required=True), desc=PROVIDER_NAME) +@API.Input.json(device=fields.String(required=True), desc=DEVICE) +@API.Input.json(version=fields.String(required=True), desc=VERSION) +@API.Input.json(metadata=fields.Dict(required=True), desc=COMPONENT_METADATA) +@API.Input.json(protocol=fields.String(required=False), desc=PROTOCOL) +@API.Input.json(components_description=fields.Dict(required=False), desc=COMPONENTS_DESCRIPTION) +def register(name, device, version, metadata, components_description=None, protocol=None): + provider = ProviderManager.get_provider(name=name, device=device, version=version, metadata=metadata, check=True) + if provider: + operator_type = ProviderManager.register_provider(provider, components_description, protocol) + return API.Output.json(message=f"{operator_type} success") + else: + return API.Output.fate_flow_exception(DeviceNotSupported(device=device)) + + +@manager.route('/query', methods=['GET']) +@API.Input.params(name=fields.String(required=False), desc=PROVIDER_NAME) +@API.Input.params(device=fields.String(required=False), desc=DEVICE) +@API.Input.params(version=fields.String(required=False), desc=VERSION) +@API.Input.params(provider_name=fields.String(required=False), desc=PROVIDER_ALL_NAME) +def query(name=None, device=None, version=None, provider_name=None): + providers = ProviderManager.query_provider(name=name, device=device, version=version, provider_name=provider_name) + return API.Output.json(data=[provider.to_human_model_dict() for provider in providers]) + + +@manager.route('/delete', methods=['POST']) +@API.Input.json(name=fields.String(required=False), desc=PROVIDER_NAME) +@API.Input.json(device=fields.String(required=False), desc=DEVICE) +@API.Input.json(version=fields.String(required=False), desc=VERSION) +@API.Input.json(provider_name=fields.String(required=False), desc=PROVIDER_ALL_NAME) +def delete(name=None, device=None, version=None, provider_name=None): + result = ProviderManager.delete_provider(name=name, device=device, version=version, provider_name=provider_name) + return API.Output.json(data=result) diff --git a/python/fate_flow/apps/client/server_app.py b/python/fate_flow/apps/client/server_app.py new file mode 100644 index 000000000..232d288e9 --- /dev/null +++ b/python/fate_flow/apps/client/server_app.py @@ -0,0 +1,93 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from webargs import fields + +from fate_flow.apps.desc import SERVER_NAME, HOST, PORT, PROTOCOL, SERVICE_NAME, URI, METHOD, PARAMS, DATA, HEADERS +from fate_flow.errors.server_error import NoFoundServer +from fate_flow.manager.service.service_manager import ServiceRegistry, ServerRegistry +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.utils.api_utils import API + + +@manager.route('/fateflow', methods=['GET']) +def fate_flow_server_info(): + datas = RuntimeConfig.SERVICE_DB.get_servers(to_dict=True) + return API.Output.json(data=datas) + + +@manager.route('/query/all', methods=['GET']) +def query_all(): + data = ServerRegistry.get_all() + return API.Output.json(data=data) + + +@manager.route('/query', methods=['GET']) +@API.Input.params(server_name=fields.String(required=True), desc=SERVER_NAME) +def query_server(server_name): + server_list = ServerRegistry.query_server_info_from_db(server_name) + if not server_list: + return API.Output.fate_flow_exception(NoFoundServer(server_name=server_name)) + return API.Output.json(data=server_list[0].to_human_model_dict()) + + +@manager.route('/registry', methods=['POST']) +@API.Input.json(server_name=fields.String(required=True), desc=SERVER_NAME) +@API.Input.json(host=fields.String(required=True), desc=HOST) +@API.Input.json(port=fields.Integer(required=True), desc=PORT) +@API.Input.json(protocol=fields.String(required=False), desc=PROTOCOL) +def register_server(server_name, host, port, protocol="http"): + server_info = ServerRegistry.register(server_name, host, port, protocol) + return API.Output.json(data=server_info) + + +@manager.route('/delete', methods=['POST']) +@API.Input.json(server_name=fields.String(required=True), desc=SERVER_NAME) +def delete_server(server_name): + status = ServerRegistry.delete_server_from_db(server_name) + return API.Output.json(message="success" if status else "failed") + + +@manager.route('/service/query', methods=['GET']) +@API.Input.params(server_name=fields.String(required=True), desc=SERVER_NAME) +@API.Input.params(service_name=fields.String(required=True), desc=SERVICE_NAME) +def query_service(server_name, service_name): + service_list = ServiceRegistry.load_service(server_name=server_name, service_name=service_name) + if not service_list: + return API.Output.fate_flow_exception(NoFoundServer(server_name=server_name)) + return API.Output.json(data=service_list[0].to_human_model_dict()) + + +@manager.route('/service/registry', methods=['POST']) +@API.Input.json(server_name=fields.String(required=True), desc=SERVER_NAME) +@API.Input.json(service_name=fields.String(required=True), desc=SERVICE_NAME) +@API.Input.json(uri=fields.String(required=True), desc=URI) +@API.Input.json(method=fields.String(required=False), desc=METHOD) +@API.Input.json(params=fields.Dict(required=False), desc=PARAMS) +@API.Input.json(data=fields.Dict(required=False), desc=DATA) +@API.Input.json(headers=fields.Dict(required=False), desc=HEADERS) +@API.Input.json(protocol=fields.String(required=False), desc=PROTOCOL) +def registry_service(server_name, service_name, uri, method="POST", params=None, data=None, headers=None, protocol="http"): + ServiceRegistry.save_service_info(server_name=server_name, service_name=service_name, uri=uri, method=method, + params=params, data=data, headers=headers, protocol=protocol) + return API.Output.json() + + +@manager.route('/service/delete', methods=['POST']) +@API.Input.json(server_name=fields.String(required=True), desc=SERVER_NAME) +@API.Input.json(service_name=fields.String(required=True), desc=SERVICE_NAME) +def delete_service(server_name, service_name): + status = ServiceRegistry.delete(server_name, service_name) + return API.Output.json(message="success" if status else "failed") diff --git a/python/fate_flow/apps/pipeline_app.py b/python/fate_flow/apps/client/site_app.py similarity index 56% rename from python/fate_flow/apps/pipeline_app.py rename to python/fate_flow/apps/client/site_app.py index 40f3f6162..9a83bfcb2 100644 --- a/python/fate_flow/apps/pipeline_app.py +++ b/python/fate_flow/apps/client/site_app.py @@ -13,16 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from flask import request +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import IsStandalone +from fate_flow.runtime.system_settings import PARTY_ID, IS_STANDALONE +from fate_flow.utils.api_utils import API -from fate_flow.utils.api_utils import get_json_result -from fate_flow.manager import pipeline_manager - -@manager.route('/dag/dependency', methods=['post']) -def pipeline_dag_dependency(): - dependency = pipeline_manager.pipeline_dag_dependency(request.json) - if dependency: - return get_json_result(retcode=0, retmsg='success', data=dependency) +@manager.route('/info/query', methods=['GET']) +def query_site_info(): + if not IS_STANDALONE: + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data={"party_id": PARTY_ID}) else: - return get_json_result(retcode=101, retmsg='') + return API.Output.fate_flow_exception(IsStandalone()) diff --git a/python/fate_flow/apps/client/table_app.py b/python/fate_flow/apps/client/table_app.py new file mode 100644 index 000000000..77b50f469 --- /dev/null +++ b/python/fate_flow/apps/client/table_app.py @@ -0,0 +1,66 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from webargs import fields + +from fate_flow.apps.desc import NAMESPACE, NAME, DISPLAY, SERVER_FILE_PATH +from fate_flow.engine import storage +from fate_flow.engine.storage import StorageEngine, DataType +from fate_flow.errors.server_error import NoFoundTable +from fate_flow.manager.outputs.data import DataManager +from fate_flow.utils.api_utils import API + +page_name = "table" + + +@manager.route('/query', methods=['GET']) +@API.Input.params(namespace=fields.String(required=True), desc=NAMESPACE) +@API.Input.params(name=fields.String(required=True), desc=NAME) +@API.Input.params(display=fields.Bool(required=False), desc=DISPLAY) +def query_table(namespace, name, display=False): + data, display_data = DataManager.get_data_info(namespace, name) + if data: + if display: + data.update({"display": display_data}) + return API.Output.json(data=data) + else: + return API.Output.fate_flow_exception(NoFoundTable(name=name, namespace=namespace)) + + +@manager.route('/delete', methods=['POST']) +@API.Input.json(namespace=fields.String(required=True), desc=NAMESPACE) +@API.Input.json(name=fields.String(required=True), desc=NAME) +def delete_table(namespace, name): + if DataManager.delete_data(namespace, name): + return API.Output.json() + else: + return API.Output.fate_flow_exception(NoFoundTable(name=name, namespace=namespace)) + + +@manager.route('/bind/path', methods=['POST']) +@API.Input.json(namespace=fields.String(required=True), desc=NAMESPACE) +@API.Input.json(name=fields.String(required=True), desc=NAME) +@API.Input.json(path=fields.String(required=True), desc=SERVER_FILE_PATH) +def bind_path(namespace, name, path): + address = storage.StorageTableMeta.create_address(storage_engine=StorageEngine.PATH, address_dict={"path": path}) + storage_meta = storage.StorageTableBase( + namespace=namespace, name=name, address=address, + engine=StorageEngine.PATH, options=None, partitions=None, + key_serdes_type=0, + value_serdes_type=0, + partitioner_type=0, + ) + storage_meta.create_meta(data_type=DataType.DATA_DIRECTORY) + return API.Output.json() diff --git a/python/fate_flow/apps/component_app.py b/python/fate_flow/apps/component_app.py deleted file mode 100644 index c870a59ee..000000000 --- a/python/fate_flow/apps/component_app.py +++ /dev/null @@ -1,311 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from flask import request - -from fate_arch.common.file_utils import get_federatedml_setting_conf_directory - -from fate_flow.component_env_utils.env_utils import get_class_object -from fate_flow.db.component_registry import ComponentRegistry -from fate_flow.db.db_models import PipelineComponentMeta -from fate_flow.model.sync_model import SyncComponent -from fate_flow.pipelined_model.pipelined_model import PipelinedModel -from fate_flow.settings import ENABLE_MODEL_STORE -from fate_flow.utils.api_utils import error_response, get_json_result, validate_request -from fate_flow.utils.detect_utils import check_config -from fate_flow.utils.job_utils import generate_job_id -from fate_flow.utils.model_utils import gen_party_model_id -from fate_flow.utils.schedule_utils import get_dsl_parser_by_version - - -@manager.route('/get', methods=['POST']) -def get_components(): - return get_json_result(data=ComponentRegistry.get_components()) - - -@manager.route('//get', methods=['POST']) -def get_component(component_name): - return get_json_result(data=ComponentRegistry.get_components().get(component_name)) - - -@manager.route('/validate', methods=['POST']) -def validate_component_param(): - if not request.json or not isinstance(request.json, dict): - return error_response(400) - - required_keys = [ - 'component_name', - 'component_module_name', - ] - config_keys = ['role'] - - dsl_version = int(request.json.get('dsl_version', 0)) - parser_class = get_dsl_parser_by_version(dsl_version) - if dsl_version == 1: - config_keys += ['role_parameters', 'algorithm_parameters'] - elif dsl_version == 2: - config_keys += ['component_parameters'] - else: - return error_response(400, 'unsupported dsl_version') - - try: - check_config(request.json, required_keys + config_keys) - except Exception as e: - return error_response(400, str(e)) - - try: - parser_class.validate_component_param( - get_federatedml_setting_conf_directory(), - {i: request.json[i] for i in config_keys}, - *[request.json[i] for i in required_keys]) - except Exception as e: - return error_response(400, str(e)) - - return get_json_result() - - -@manager.route('/hetero/merge', methods=['POST']) -@validate_request( - 'model_id', 'model_version', 'guest_party_id', 'host_party_ids', - 'component_name', 'model_type', 'output_format', -) -def hetero_model_merge(): - request_data = request.json - - if ENABLE_MODEL_STORE: - sync_component = SyncComponent( - role='guest', - party_id=request_data['guest_party_id'], - model_id=request_data['model_id'], - model_version=request_data['model_version'], - component_name=request_data['component_name'], - ) - if not sync_component.local_exists() and sync_component.remote_exists(): - sync_component.download() - - for party_id in request_data['host_party_ids']: - sync_component = SyncComponent( - role='host', - party_id=party_id, - model_id=request_data['model_id'], - model_version=request_data['model_version'], - component_name=request_data['component_name'], - ) - if not sync_component.local_exists() and sync_component.remote_exists(): - sync_component.download() - - model = PipelinedModel( - gen_party_model_id( - request_data['model_id'], - 'guest', - request_data['guest_party_id'], - ), - request_data['model_version'], - ).read_component_model( - request_data['component_name'], - output_json=True, - ) - - guest_param = None - guest_meta = None - - for k, v in model.items(): - if k.endswith('Param'): - guest_param = v - elif k.endswith('Meta'): - guest_meta = v - else: - return error_response(400, f'Unknown guest model key: "{k}".') - - if guest_param is None or guest_meta is None: - return error_response(400, 'Invalid guest model.') - - host_params = [] - host_metas = [] - - for party_id in request_data['host_party_ids']: - model = PipelinedModel( - gen_party_model_id( - request_data['model_id'], - 'host', - party_id, - ), - request_data['model_version'], - ).read_component_model( - request_data['component_name'], - output_json=True, - ) - - for k, v in model.items(): - if k.endswith('Param'): - host_params.append(v) - elif k.endswith('Meta'): - host_metas.append(v) - else: - return error_response(400, f'Unknown host model key: "{k}".') - - if not host_params or not host_metas or len(host_params) != len(host_metas): - return error_response(400, 'Invalid host models.') - - data = get_class_object('hetero_model_merge')( - guest_param, guest_meta, - host_params, host_metas, - request_data['model_type'], - request_data['output_format'], - request_data.get('target_name', 'y'), - request_data.get('host_rename', False), - request_data.get('include_guest_coef', False), - ) - return get_json_result(data=data) - - -@manager.route('/woe_array/extract', methods=['POST']) -@validate_request( - 'model_id', 'model_version', 'role', 'party_id', 'component_name', -) -def woe_array_extract(): - if request.json['role'] != 'guest': - return error_response(400, 'Only support guest role.') - - if ENABLE_MODEL_STORE: - sync_component = SyncComponent( - role=request.json['role'], - party_id=request.json['party_id'], - model_id=request.json['model_id'], - model_version=request.json['model_version'], - component_name=request.json['component_name'], - ) - if not sync_component.local_exists() and sync_component.remote_exists(): - sync_component.download() - - model = PipelinedModel( - gen_party_model_id( - request.json['model_id'], - request.json['role'], - request.json['party_id'], - ), - request.json['model_version'], - ).read_component_model( - request.json['component_name'], - output_json=True, - ) - - param = None - meta = None - - for k, v in model.items(): - if k.endswith('Param'): - param = v - elif k.endswith('Meta'): - meta = v - else: - return error_response(400, f'Unknown model key: "{k}".') - - if param is None or meta is None: - return error_response(400, 'Invalid model.') - - data = get_class_object('extract_woe_array_dict')(param) - return get_json_result(data=data) - - -@manager.route('/woe_array/merge', methods=['POST']) -@validate_request( - 'model_id', 'model_version', 'role', 'party_id', 'component_name', 'woe_array', -) -def woe_array_merge(): - if request.json['role'] != 'host': - return error_response(400, 'Only support host role.') - - pipelined_model = PipelinedModel( - gen_party_model_id( - request.json['model_id'], - request.json['role'], - request.json['party_id'], - ), - request.json['model_version'], - ) - - query = pipelined_model.pipelined_component.get_define_meta_from_db( - PipelineComponentMeta.f_component_name == request.json['component_name'], - ) - if not query: - return error_response(404, 'Component not found.') - query = query[0] - - if ENABLE_MODEL_STORE: - sync_component = SyncComponent( - role=query.f_role, - party_id=query.f_party_id, - model_id=query.f_model_id, - model_version=query.f_model_version, - component_name=query.f_component_name, - ) - if not sync_component.local_exists() and sync_component.remote_exists(): - sync_component.download() - - model = pipelined_model._read_component_model( - query.f_component_name, - query.f_model_alias, - ) - - for model_name, ( - buffer_name, - buffer_string, - buffer_dict, - ) in model.items(): - if model_name.endswith('Param'): - string_merged, dict_merged = get_class_object('merge_woe_array_dict')( - buffer_name, - buffer_string, - buffer_dict, - request.json['woe_array'], - ) - model[model_name] = ( - buffer_name, - string_merged, - dict_merged, - ) - break - - pipelined_model = PipelinedModel( - pipelined_model.party_model_id, - generate_job_id() - ) - - pipelined_model.save_component_model( - query.f_component_name, - query.f_component_module_name, - query.f_model_alias, - model, - query.f_run_parameters, - ) - - if ENABLE_MODEL_STORE: - sync_component = SyncComponent( - role=query.f_role, - party_id=query.f_party_id, - model_id=query.f_model_id, - model_version=pipelined_model.model_version, - component_name=query.f_component_name, - ) - sync_component.upload() - - return get_json_result(data={ - 'role': query.f_role, - 'party_id': query.f_party_id, - 'model_id': query.f_model_id, - 'model_version': pipelined_model.model_version, - 'component_name': query.f_component_name, - }) diff --git a/python/fate_flow/apps/data_access_app.py b/python/fate_flow/apps/data_access_app.py deleted file mode 100644 index f3240ab56..000000000 --- a/python/fate_flow/apps/data_access_app.py +++ /dev/null @@ -1,215 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from uuid import uuid1 -from pathlib import Path - -from flask import request - -from fate_flow.entity.run_status import StatusSet -from fate_flow.entity import JobConfigurationBase -from fate_arch import storage -from fate_arch.common import FederatedMode -from fate_arch.common.base_utils import json_loads -from fate_flow.settings import UPLOAD_DATA_FROM_CLIENT -from fate_flow.utils.api_utils import get_json_result, error_response -from fate_flow.utils import detect_utils, job_utils -from fate_flow.scheduler.dag_scheduler import DAGScheduler -from fate_flow.operation.job_saver import JobSaver - - -page_name = 'data' - - -@manager.route('/', methods=['post']) -def download_upload(access_module): - job_id = job_utils.generate_job_id() - - if access_module == "upload" and UPLOAD_DATA_FROM_CLIENT and not (request.json and request.json.get("use_local_data") == 0): - file = request.files['file'] - filename = Path(job_utils.get_job_directory(job_id), 'fate_upload_tmp', uuid1().hex) - filename.parent.mkdir(parents=True, exist_ok=True) - - try: - file.save(str(filename)) - except Exception as e: - try: - filename.unlink() - except FileNotFoundError: - pass - - return error_response(500, f'Save file error: {e}') - - job_config = request.args.to_dict() or request.form.to_dict() - if "namespace" not in job_config or "table_name" not in job_config: - # higher than version 1.5.1, support eggroll run parameters - job_config = json_loads(list(job_config.keys())[0]) - - job_config['file'] = str(filename) - else: - job_config = request.json - - required_arguments = ['namespace', 'table_name'] - if access_module == 'upload': - required_arguments.extend(['file', 'head', 'partition']) - elif access_module == 'download': - required_arguments.extend(['output_path']) - elif access_module == 'writer': - pass - else: - return error_response(400, f'Cannot support this operating: {access_module}') - - detect_utils.check_config(job_config, required_arguments=required_arguments) - data = {} - # compatibility - if "table_name" in job_config: - job_config["name"] = job_config["table_name"] - for _ in ["head", "partition", "drop", "extend_sid", "auto_increasing_sid"]: - if _ in job_config: - if _ == "false": - job_config[_] = False - elif _ == "true": - job_config[_] = True - else: - job_config[_] = int(job_config[_]) - if access_module == "upload": - if int(job_config.get('drop', 0)) > 0: - job_config["destroy"] = True - else: - job_config["destroy"] = False - data['table_name'] = job_config["table_name"] - data['namespace'] = job_config["namespace"] - data_table_meta = storage.StorageTableMeta(name=job_config["table_name"], namespace=job_config["namespace"]) - if data_table_meta and not job_config["destroy"]: - return get_json_result(retcode=100, - retmsg='The data table already exists.' - 'If you still want to continue uploading, please add the parameter --drop') - job_dsl, job_runtime_conf = gen_data_access_job_config(job_config, access_module) - submit_result = DAGScheduler.submit(JobConfigurationBase(**{'dsl': job_dsl, 'runtime_conf': job_runtime_conf}), job_id=job_id) - data.update(submit_result) - return get_json_result(job_id=job_id, data=data) - - -@manager.route('/upload/history', methods=['POST']) -def upload_history(): - request_data = request.json - if request_data.get('job_id'): - tasks = JobSaver.query_task(component_name='upload_0', status=StatusSet.SUCCESS, job_id=request_data.get('job_id'), run_on_this_party=True) - else: - tasks = JobSaver.query_task(component_name='upload_0', status=StatusSet.SUCCESS, run_on_this_party=True) - limit = request_data.get('limit') - if not limit: - tasks = tasks[-1::-1] - else: - tasks = tasks[-1:-limit - 1:-1] - jobs_run_conf = job_utils.get_upload_job_configuration_summary(upload_tasks=tasks) - data = get_upload_info(jobs_run_conf=jobs_run_conf) - return get_json_result(retcode=0, retmsg='success', data=data) - - -def get_upload_info(jobs_run_conf): - data = [] - - for job_id, job_run_conf in jobs_run_conf.items(): - info = {} - table_name = job_run_conf["name"] - namespace = job_run_conf["namespace"] - table_meta = storage.StorageTableMeta(name=table_name, namespace=namespace) - if table_meta: - partition = job_run_conf["partition"] - info["upload_info"] = { - "table_name": table_name, - "namespace": namespace, - "partition": partition, - 'upload_count': table_meta.get_count() - } - info["notes"] = job_run_conf["notes"] - info["schema"] = table_meta.get_schema() - data.append({job_id: info}) - return data - - -def gen_data_access_job_config(config_data, access_module): - job_runtime_conf = { - "initiator": {}, - "job_parameters": {"common": {}}, - "role": {}, - "component_parameters": {"role": {"local": {"0": {}}}} - } - initiator_role = "local" - initiator_party_id = config_data.get('party_id', 0) - job_runtime_conf["initiator"]["role"] = initiator_role - job_runtime_conf["initiator"]["party_id"] = initiator_party_id - job_parameters_fields = {"task_cores", "eggroll_run", "spark_run", "computing_engine", "storage_engine", "federation_engine"} - for _ in job_parameters_fields: - if _ in config_data: - job_runtime_conf["job_parameters"]["common"][_] = config_data[_] - job_runtime_conf["job_parameters"]["common"]["federated_mode"] = FederatedMode.SINGLE - job_runtime_conf["role"][initiator_role] = [initiator_party_id] - job_dsl = { - "components": {} - } - - if access_module == 'upload': - parameters = { - "head", - "partition", - "file", - "namespace", - "name", - "id_delimiter", - "storage_engine", - "storage_address", - "destroy", - "extend_sid", - "auto_increasing_sid", - "block_size", - "schema", - "with_meta", - "meta" - } - update_config(job_runtime_conf, job_dsl, initiator_role, parameters, access_module, config_data) - - if access_module == 'download': - parameters = { - "delimiter", - "output_path", - "namespace", - "name" - } - update_config(job_runtime_conf, job_dsl, initiator_role, parameters, access_module, config_data) - - if access_module == 'writer': - parameters = { - "namespace", - "name", - "storage_engine", - "address", - "output_namespace", - "output_name" - } - update_config(job_runtime_conf, job_dsl, initiator_role, parameters, access_module, config_data) - return job_dsl, job_runtime_conf - - -def update_config(job_runtime_conf, job_dsl, initiator_role, parameters, access_module, config_data): - job_runtime_conf["component_parameters"]['role'][initiator_role]["0"][f"{access_module}_0"] = {} - for p in parameters: - if p in config_data: - job_runtime_conf["component_parameters"]['role'][initiator_role]["0"][f"{access_module}_0"][p] = config_data[p] - job_runtime_conf['dsl_version'] = 2 - job_dsl["components"][f"{access_module}_0"] = { - "module": access_module.capitalize() - } diff --git a/python/fate_flow/apps/desc.py b/python/fate_flow/apps/desc.py new file mode 100644 index 000000000..6ee7fdcff --- /dev/null +++ b/python/fate_flow/apps/desc.py @@ -0,0 +1,100 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# job +DAG_SCHEMA = "Definition and configuration of jobs, including the configuration of multiple tasks" +USER_NAME = "Username provided by the upper-level system" +JOB_ID = "Job ID" +ROLE = "Role of the participant: guest/host/arbiter/local" +STATUS = "Status of the job or task" +LIMIT = "Limit of rows or entries" +PAGE = "Page number" +DESCRIPTION = "Description information" +PARTNER = "Participant information" +ORDER_BY = "Field name for sorting" +ORDER = "Sorting order: asc/desc" + +# task +TASK_NAME = "Task name" +TASK_ID = "Task ID" +TASK_VERSION = "Task version" +NODES = "Tags and customizable information for tasks" + +# data +SERVER_FILE_PATH = "File path on the server" +SERVER_DIR_PATH = "Directory path on the server" +HEAD = "Whether the first row of the file is the data's head" +PARTITIONS = "Number of data partitions" +META = "Metadata of the data" +EXTEND_SID = "Whether to automatically fill a column as data row ID" +NAMESPACE = "Namespace of the data table" +NAME = "Name of the data table" +SITE_NAME = "Site name" +DATA_WAREHOUSE = "Data output, content like: {name: xxx, namespace: xxx}" +DROP = "Whether to destroy data if it already exists" +DOWNLOAD_HEADER = "Whether to download the data's head as the first row" + +# output +FILTERS = "Filter conditions" +OUTPUT_KEY = "Primary key for output data or model of the task" + +# table +DISPLAY = "Whether to return preview data" + +# server +SERVER_NAME = "Server name" +SERVICE_NAME = "Service name" +HOST = "Host IP" +PORT = "Service port" +PROTOCOL = "Protocol: fate/bfia,etc." +URI = "Service path" +METHOD = "Request method: POST/GET, etc." +PARAMS = "Request header parameters" +DATA = "Request body parameters" +HEADERS = "Request headers" + +# provider +PROVIDER_NAME = "Component provider name" +DEVICE = "Component running mode" +VERSION = "Component version" +COMPONENT_METADATA = "Detailed information about component registration" +COMPONENTS_DESCRIPTION = "Components description" +PROVIDER_ALL_NAME = "Registered algorithm full name, provider + ':' + version + '@' + running mode, e.g., fate:2.0.0@local" + +# permission +PERMISSION_APP_ID = "App ID" +PERMISSION_ROLE = "Permission name" +COMPONENT = "Component name" +DATASET = "List of datasets" + +# log +LOG_TYPE = "Log level or type" +INSTANCE_ID = "Instance ID of the FATE Flow service" +BEGIN = "Starting line number" +END = "Ending line number" + +# site +PARTY_ID = "Site ID" + +# model +MODEL_ID = "Model ID" +MODEL_VERSION = "Model version" + +# app +APP_NAME = "App name for the client" +APP_ID = "App ID for the client" +SITE_APP_ID = "App ID for the site" +SITE_APP_TOKEN = "App token for the site" diff --git a/python/fate_flow/apps/desc_zh.py b/python/fate_flow/apps/desc_zh.py new file mode 100644 index 000000000..a82b4602e --- /dev/null +++ b/python/fate_flow/apps/desc_zh.py @@ -0,0 +1,100 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# job +DAG_SCHEMA = "作业的定义和配置，包括多个任务的配置" +USER_NAME = "上层系统所提供的用户名" +JOB_ID = "作业ID" +ROLE = "参与方的角色: guest/host/arbiter/local" +STATUS = "作业或者任务的状态" +LIMIT = "限制条数或者行数" +PAGE = "页码数" +DESCRIPTION = "描述信息" +PARTNER = "参与方信息" +ORDER_BY = "排序的字段名" +ORDER = "排序方式：asc/desc" + +# task +TASK_NAME = "任务名称" +TASK_ID = "任务ID" +TASK_VERSION = "任务版本" +NODES = "任务的标签等信息，用户可自定义化" + +# data +SERVER_FILE_PATH = "服务器上的文件路径" +SERVER_DIR_PATH = "服务器上的目录路径" +HEAD = "文件的第一行是否为数据的Head" +PARTITIONS = "数据分区数量" +META = "数据的元信息" +EXTEND_SID = "是否需要自动填充一列作为数据行id" +NAMESPACE = "数据表的命名空间" +NAME = "数据表名" +DATA_WAREHOUSE = "数据输出，内容如:{name: xxx, namespace: xxx}" +DROP = "当数据存在时是否需要先销毁" +DOWNLOAD_HEADER = "是否需要下载数据的Head作为第一行" + +# output +FILTERS = "过滤条件" +OUTPUT_KEY = "任务的输出数据或者模型的主键" + +# table +DISPLAY = "是否需要返回预览数据" + +# server +SERVER_NAME = "服务器名称" +SERVICE_NAME = "服务名称" +HOST = "主机ip" +PORT = "服务端口" +PROTOCOL = "协议：http/https" +URI = "服务路径" +METHOD = "请求方式：POST/GET等" +PARAMS = "请求头参数" +DATA = "请求体参数" +HEADERS = "请求头" + +# provider +PROVIDER_NAME = "组件提供方名称" +DEVICE = "组件运行模式" +VERSION = "组件版本" +COMPONENT_METADATA = "组件注册详细信息" +COMPONENTS_DESCRIPTION = "组件描述" +PROVIDER_ALL_NAME = "注册的算法全名，提供方+':'+版本+'@'+运行模式，如: fate:2.0.0@local" + +# permission +PERMISSION_APP_ID = "App id" +PERMISSION_ROLE = "权限名称" +COMPONENT = "组件名" +DATASET = "数据集列表" + +# log +LOG_TYPE = "日志等级或类型" +INSTANCE_ID = "FATE Flow服务的实例ID" +BEGIN = "起始行号" +END = "结尾行号" + + +# site +PARTY_ID = "站点ID" + +# model +MODEL_ID = "模型id" +MODEL_VERSION = "模型版本" + +# app +APP_NAME = "客户端的app名称" +APP_ID = "客户端的app-id" +SITE_APP_ID = "站点的app-id" +SITE_APP_TOKEN = "站点的app-token" diff --git a/python/fate_flow/apps/info_app.py b/python/fate_flow/apps/info_app.py deleted file mode 100644 index e8eb3de25..000000000 --- a/python/fate_flow/apps/info_app.py +++ /dev/null @@ -1,96 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import socket - -from flask import request -from flask.json import jsonify - -from fate_arch.common import FederatedMode - -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.db.service_registry import ServerRegistry -from fate_flow.settings import API_VERSION, GRPC_PORT, HOST, HTTP_PORT, PARTY_ID -from fate_flow.utils.api_utils import error_response, federated_api, get_json_result - - -@manager.route('/common', methods=['POST']) -def get_common_info(): - return get_json_result(data={ - 'version': RuntimeConfig.get_env('FATE'), - 'host': HOST, - 'http_port': HTTP_PORT, - 'grpc_port': GRPC_PORT, - 'party_id': PARTY_ID, - }) - - -@manager.route('/fateboard', methods=['POST']) -def get_fateboard_info(): - host = ServerRegistry.FATEBOARD.get('host') - port = ServerRegistry.FATEBOARD.get('port') - if not host or not port: - return error_response(404, 'fateboard is not configured') - - return get_json_result(data={ - 'host': host, - 'port': port, - }) - - -# TODO: send greetings message using grpc protocol -@manager.route('/eggroll', methods=['POST']) -def get_eggroll_info(): - conf = ServerRegistry.FATE_ON_EGGROLL['rollsite'] - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - r = s.connect_ex((conf['host'], conf['port'])) - if r != 0: - return error_response(503) - - return error_response(200) - - -@manager.route('/version', methods=['POST']) -@app.route(f'/{API_VERSION}/version/get', methods=['POST']) -def get_version(): - module = request.json['module'] if isinstance(request.json, dict) and request.json.get('module') else 'FATE' - version = RuntimeConfig.get_env(module) - if version is None: - return error_response(404, f'unknown module {module}') - - return get_json_result(data={ - module: version, - 'API': API_VERSION, - }) - - -@manager.route('/party/', methods=['POST']) -def get_party_info(dest_party_id): - response = federated_api( - 'party_info', 'POST', '/info/common', - PARTY_ID, dest_party_id, '', - {}, FederatedMode.MULTIPLE, - ) - return jsonify(response) - - -@manager.route('/party//', methods=['POST']) -def get_party_info_from_another_party(proxy_party_id, dest_party_id): - response = federated_api( - 'party_info', 'POST', f'/info/party/{dest_party_id}', - PARTY_ID, proxy_party_id, '', - {}, FederatedMode.MULTIPLE, - ) - return jsonify(response) diff --git a/python/fate_flow/apps/job_app.py b/python/fate_flow/apps/job_app.py deleted file mode 100644 index ee78cea84..000000000 --- a/python/fate_flow/apps/job_app.py +++ /dev/null @@ -1,368 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import io -import json -import os -import tarfile - -from flask import abort, request, send_file - -from fate_arch.common.base_utils import json_dumps, json_loads -from fate_flow.controller.job_controller import JobController -from fate_flow.entity import JobConfigurationBase, RetCode -from fate_flow.entity.run_status import FederatedSchedulingStatusCode, JobStatus -from fate_flow.operation.job_clean import JobClean -from fate_flow.operation.job_saver import JobSaver -from fate_flow.operation.job_tracker import Tracker -from fate_flow.scheduler.dag_scheduler import DAGScheduler -from fate_flow.scheduler.federated_scheduler import FederatedScheduler -from fate_flow.settings import TEMP_DIRECTORY, stat_logger -from fate_flow.utils import job_utils, log_utils, schedule_utils, api_utils -from fate_flow.utils.api_utils import error_response, get_json_result -from fate_flow.utils.config_adapter import JobRuntimeConfigAdapter -from fate_flow.utils.log_utils import schedule_logger - - -@manager.route('/submit', methods=['POST']) -def submit_job(): - submit_result = DAGScheduler.submit(JobConfigurationBase(**request.json)) - return get_json_result(retcode=submit_result["code"], retmsg=submit_result["message"], - job_id=submit_result["job_id"], - data=submit_result if submit_result["code"] == RetCode.SUCCESS else None) - - -@manager.route('/stop', methods=['POST']) -def stop_job(): - job_id = request.json.get('job_id') - stop_status = request.json.get("stop_status", "canceled") - jobs = JobSaver.query_job(job_id=job_id) - if jobs: - schedule_logger(job_id).info(f"stop job on this party") - kill_status, kill_details = JobController.stop_jobs(job_id=job_id, stop_status=stop_status) - schedule_logger(job_id).info(f"stop job on this party status {kill_status}") - schedule_logger(job_id).info(f"request stop job to {stop_status}") - status_code, response = FederatedScheduler.request_stop_job(job=jobs[0], stop_status=stop_status, command_body=jobs[0].to_dict()) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - return get_json_result(retcode=RetCode.SUCCESS, retmsg=f"stop job on this party {'success' if kill_status else 'failed'}; stop job on all party success") - else: - return get_json_result(retcode=RetCode.OPERATING_ERROR, retmsg=f"stop job on this party {kill_status}", data=response) - else: - schedule_logger(job_id).info(f"can not found job to stop") - return get_json_result(retcode=RetCode.DATA_ERROR, retmsg="can not found job") - - -@manager.route('/rerun', methods=['POST']) -def rerun_job(): - job_id = request.json.get("job_id") - jobs = JobSaver.query_job(job_id=job_id) - if jobs: - status_code, response = FederatedScheduler.request_rerun_job(job=jobs[0], command_body=request.json) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - return get_json_result(retcode=RetCode.SUCCESS, retmsg="rerun job success") - else: - return get_json_result(retcode=RetCode.OPERATING_ERROR, retmsg="rerun job failed:\n{}".format(json_dumps(response))) - else: - return get_json_result(retcode=RetCode.DATA_ERROR, retmsg="can not found job") - - -@manager.route('/query', methods=['POST']) -def query_job(): - jobs = JobSaver.query_job(**request.json) - if not jobs: - return get_json_result(retcode=0, retmsg='no job could be found', data=[]) - return get_json_result(retcode=0, retmsg='success', data=[job.to_dict() for job in jobs]) - - -@manager.route('/list/job', methods=['POST']) -def list_job(): - limit, offset = parse_limit_and_offset() - - query = { - 'tag': ('!=', 'submit_failed'), - } - - for i in ('job_id', 'description'): - if request.json.get(i) is not None: - query[i] = ('contains', request.json[i]) - if request.json.get('party_id') is not None: - try: - query['party_id'] = int(request.json['party_id']) - except Exception: - return error_response(400, f"Invalid parameter 'party_id'.") - query['party_id'] = ('contains', query['party_id']) - if request.json.get('partner') is not None: - query['roles'] = ('contains', query['partner']) - - for i in ('role', 'status'): - if request.json.get(i) is None: - continue - - if isinstance(request.json[i], str): - request.json[i] = [request.json[i]] - if not isinstance(request.json[i], list): - return error_response(400, f"Invalid parameter '{i}'.") - request.json[i] = set(request.json[i]) - - for j in request.json[i]: - if j not in valid_query_parameters[i]: - return error_response(400, f"Invalid parameter '{i}'.") - query[i] = ('in_', request.json[i]) - - jobs, count = job_utils.list_job(limit, offset, query, parse_order_by(('create_time', 'desc'))) - jobs = [job.to_human_model_dict() for job in jobs] - - for job in jobs: - job['party_id'] = int(job['party_id']) - - job['partners'] = set() - for i in ('guest', 'host', 'arbiter'): - job['partners'].update(job['roles'].get(i, [])) - job['partners'].discard(job['party_id']) - job['partners'] = sorted(job['partners']) - - return get_json_result(data={ - 'jobs': jobs, - 'count': count, - }) - - -@manager.route('/update', methods=['POST']) -def update_job(): - job_info = request.json - jobs = JobSaver.query_job(job_id=job_info['job_id'], party_id=job_info['party_id'], role=job_info['role']) - if not jobs: - return get_json_result(retcode=101, retmsg='find job failed') - else: - JobSaver.update_job(job_info={'description': job_info.get('notes', ''), 'job_id': job_info['job_id'], 'role': job_info['role'], - 'party_id': job_info['party_id']}) - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('/report', methods=['POST']) -def job_report(): - jobs = JobSaver.query_job(**request.json) - tasks = JobSaver.query_task(**request.json) - if not tasks or not jobs: - return get_json_result(retcode=101, retmsg='find task failed') - return get_json_result(retcode=0, retmsg='success', data=job_utils.task_report(jobs, tasks)) - - -@manager.route('/parameter/update', methods=['POST']) -@api_utils.validate_request("job_id") -def update_parameters(): - job_info = request.json - component_parameters = job_info.pop("component_parameters", None) - job_parameters = job_info.pop("job_parameters", None) - job_info["is_initiator"] = True - jobs = JobSaver.query_job(**job_info) - if not jobs: - return get_json_result(retcode=RetCode.DATA_ERROR, retmsg=log_utils.failed_log(f"query job by {job_info}")) - else: - retcode, retdata = DAGScheduler.update_parameters(jobs[0], job_parameters, component_parameters) - return get_json_result(retcode=retcode, data=retdata) - - -@manager.route('/config', methods=['POST']) -def job_config(): - jobs = JobSaver.query_job(**request.json) - if not jobs: - return get_json_result(retcode=101, retmsg='find job failed') - else: - job = jobs[0] - response_data = dict() - response_data['job_id'] = job.f_job_id - response_data['dsl'] = job.f_dsl - response_data['runtime_conf'] = job.f_runtime_conf - response_data['train_runtime_conf'] = job.f_train_runtime_conf - - adapter = JobRuntimeConfigAdapter(job.f_runtime_conf) - job_parameters = adapter.get_common_parameters().to_dict() - response_data['model_info'] = {'model_id': job_parameters.get('model_id'), - 'model_version': job_parameters.get('model_version')} - return get_json_result(retcode=0, retmsg='success', data=response_data) - - -def check_job_log_dir(): - job_id = str(request.json['job_id']) - job_log_dir = job_utils.get_job_log_directory(job_id=job_id) - - if not os.path.exists(job_log_dir): - abort(error_response(404, f"Log file path: '{job_log_dir}' not found. Please check if the job id is valid.")) - - return job_id, job_log_dir - - -@manager.route('/log/download', methods=['POST']) -@api_utils.validate_request('job_id') -def job_log_download(): - job_id, job_log_dir = check_job_log_dir() - - memory_file = io.BytesIO() - tar = tarfile.open(fileobj=memory_file, mode='w:gz') - for root, dir, files in os.walk(job_log_dir): - for file in files: - full_path = os.path.join(root, file) - rel_path = os.path.relpath(full_path, job_log_dir) - tar.add(full_path, rel_path) - - tar.close() - memory_file.seek(0) - return send_file(memory_file, attachment_filename=f'job_{job_id}_log.tar.gz', as_attachment=True) - - -@manager.route('/log/path', methods=['POST']) -@api_utils.validate_request('job_id') -def job_log_path(): - job_id, job_log_dir = check_job_log_dir() - - return get_json_result(data={"logs_directory": job_log_dir}) - - -@manager.route('/task/query', methods=['POST']) -def query_task(): - tasks = JobSaver.query_task(**request.json) - if not tasks: - return get_json_result(retcode=101, retmsg='find task failed') - return get_json_result(retcode=0, retmsg='success', data=[task.to_dict() for task in tasks]) - - -@manager.route('/list/task', methods=['POST']) -def list_task(): - limit, offset = parse_limit_and_offset() - - query = {} - for i in ('job_id', 'role', 'party_id', 'component_name'): - if request.json.get(i) is not None: - query[i] = request.json[i] - if query.get('role') is not None: - if query['role'] not in valid_query_parameters['role']: - return error_response(400, f"Invalid parameter 'role'.") - if query.get('party_id') is not None: - try: - query['party_id'] = int(query['party_id']) - except Exception: - return error_response(400, f"Invalid parameter 'party_id'.") - - tasks, count = job_utils.list_task(limit, offset, query, parse_order_by(('create_time', 'asc'))) - return get_json_result(data={ - 'tasks': [task.to_human_model_dict() for task in tasks], - 'count': count, - }) - - -@manager.route('/data/view/query', methods=['POST']) -def query_component_output_data_info(): - output_data_infos = Tracker.query_output_data_infos(**request.json) - if not output_data_infos: - return get_json_result(retcode=101, retmsg='find data view failed') - return get_json_result(retcode=0, retmsg='success', data=[output_data_info.to_dict() for output_data_info in output_data_infos]) - - -@manager.route('/clean', methods=['POST']) -def clean_job(): - JobClean.start_clean_job(**request.json) - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('/clean/queue', methods=['POST']) -def clean_queue(): - jobs = JobSaver.query_job(is_initiator=True, status=JobStatus.WAITING) - clean_status = {} - for job in jobs: - status_code, response = FederatedScheduler.request_stop_job(job=job, stop_status=JobStatus.CANCELED) - clean_status[job.f_job_id] = status_code - return get_json_result(retcode=0, retmsg='success', data=clean_status) - - -@manager.route('/dsl/generate', methods=['POST']) -def dsl_generator(): - data = request.json - cpn_str = data.get("cpn_str", "") - try: - if not cpn_str: - raise Exception("Component list should not be empty.") - if isinstance(cpn_str, list): - cpn_list = cpn_str - else: - if (cpn_str.find("/") and cpn_str.find("\\")) != -1: - raise Exception("Component list string should not contain '/' or '\\'.") - cpn_str = cpn_str.replace(" ", "").replace("\n", "").strip(",[]") - cpn_list = cpn_str.split(",") - train_dsl = json_loads(data.get("train_dsl")) - parser = schedule_utils.get_dsl_parser_by_version(data.get("version", "2")) - predict_dsl = parser.deploy_component(cpn_list, train_dsl) - - if data.get("filename"): - os.makedirs(TEMP_DIRECTORY, exist_ok=True) - temp_filepath = os.path.join(TEMP_DIRECTORY, data.get("filename")) - with open(temp_filepath, "w") as fout: - fout.write(json.dumps(predict_dsl, indent=4)) - return send_file(open(temp_filepath, 'rb'), as_attachment=True, attachment_filename=data.get("filename")) - return get_json_result(data=predict_dsl) - except Exception as e: - stat_logger.exception(e) - return error_response(210, "DSL generating failed. For more details, " - "please check logs/fate_flow/fate_flow_stat.log.") - - -@manager.route('/url/get', methods=['POST']) -@api_utils.validate_request('job_id', 'role', 'party_id') -def get_url(): - request_data = request.json - jobs = JobSaver.query_job(job_id=request_data.get('job_id'), role=request_data.get('role'), - party_id=request_data.get('party_id')) - if jobs: - board_urls = [] - for job in jobs: - board_url = job_utils.get_board_url(job.f_job_id, job.f_role, job.f_party_id) - board_urls.append(board_url) - return get_json_result(data={'board_url': board_urls}) - else: - return get_json_result(retcode=101, retmsg='no found job') - - -def parse_limit_and_offset(): - try: - limit = int(request.json.get('limit', 0)) - page = int(request.json.get('page', 1)) - 1 - except Exception: - abort(error_response(400, f"Invalid parameter 'limit' or 'page'.")) - - return limit, limit * page - - -def parse_order_by(default=None): - order_by = [] - - if request.json.get('order_by') is not None: - if request.json['order_by'] not in valid_query_parameters['order_by']: - abort(error_response(400, f"Invalid parameter 'order_by'.")) - order_by.append(request.json['order_by']) - - if request.json.get('order') is not None: - if request.json['order'] not in valid_query_parameters['order']: - abort(error_response(400, f"Invalid parameter order 'order'.")) - order_by.append(request.json['order']) - - return order_by or default - - -valid_query_parameters = { - 'role': {'guest', 'host', 'arbiter', 'local'}, - 'status': {'success', 'running', 'waiting', 'failed', 'canceled'}, - 'order_by': {'job_id', 'task_version', 'create_time', 'start_time', 'end_time', 'elapsed'}, - 'order': {'asc', 'desc'}, -} diff --git a/python/fate_flow/apps/key_app.py b/python/fate_flow/apps/key_app.py deleted file mode 100644 index 39372e027..000000000 --- a/python/fate_flow/apps/key_app.py +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from flask import request - -from fate_flow.db.key_manager import RsaKeyManager -from fate_flow.entity.types import SiteKeyName -from fate_flow.utils.api_utils import get_json_result - - -@manager.route('/public/save', methods=['POST']) -def save_public_key(): - request_conf = request.json - result = RsaKeyManager.create_or_update(request_conf.get("party_id"), request_conf.get("key")) - return get_json_result(data=result) - - -@manager.route('/query', methods=['POST']) -def query_public_key(): - request_conf = request.json - data = RsaKeyManager.get_key(request_conf.get("party_id"), key_name=request_conf.get("key_name", SiteKeyName.PUBLIC.value)) - return get_json_result(data=data) - - -@manager.route('/public/delete', methods=['POST']) -def delete_public_key(): - request_conf = request.json - RsaKeyManager.delete(request_conf.get("party_id")) - return get_json_result() \ No newline at end of file diff --git a/python/fate_flow/apps/log_app.py b/python/fate_flow/apps/log_app.py deleted file mode 100644 index ce286e946..000000000 --- a/python/fate_flow/apps/log_app.py +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from flask import request - -from fate_flow.utils.api_utils import get_json_result, cluster_route -from fate_flow.utils.log_sharing_utils import LogCollector - - -@manager.route('/size', methods=['POST']) -@cluster_route -def get_log_size(): - request_data = request.json - data = LogCollector(**request_data).get_size() - return get_json_result(retcode=0, retmsg='success', data={"size": data}) - - -@manager.route('/cat', methods=['POST']) -@cluster_route -def get_log(): - request_data = request.json - data = LogCollector(**request_data).cat_log(request_data.get("begin"), request_data.get("end")) - return get_json_result(retcode=0, retmsg='success', data=data) diff --git a/python/fate_flow/apps/model_app.py b/python/fate_flow/apps/model_app.py deleted file mode 100644 index f191f5c26..000000000 --- a/python/fate_flow/apps/model_app.py +++ /dev/null @@ -1,899 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os -import shutil -from copy import deepcopy -from uuid import uuid1 - -import peewee -from flask import abort, request, send_file - -from fate_arch.common import FederatedMode -from fate_arch.common.base_utils import json_dumps, json_loads - -from fate_flow.db.db_models import ( - DB, ModelTag, PipelineComponentMeta, Tag, - MachineLearningModelInfo as MLModel, -) -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.db.service_registry import ServerRegistry -from fate_flow.entity import JobConfigurationBase -from fate_flow.entity.types import ModelOperation, TagOperation -from fate_flow.model.sync_model import SyncComponent, SyncModel -from fate_flow.pipelined_model import deploy_model, migrate_model, publish_model -from fate_flow.pipelined_model.pipelined_model import PipelinedModel -from fate_flow.scheduler.dag_scheduler import DAGScheduler -from fate_flow.settings import ENABLE_MODEL_STORE, IS_STANDALONE, TEMP_DIRECTORY, stat_logger -from fate_flow.utils import detect_utils, job_utils, model_utils -from fate_flow.utils.api_utils import ( - error_response, federated_api, get_json_result, - send_file_in_mem, validate_request, -) -from fate_flow.utils.base_utils import compare_version -from fate_flow.utils.config_adapter import JobRuntimeConfigAdapter -from fate_flow.utils.job_utils import PIPELINE_COMPONENT_NAME -from fate_flow.utils.schedule_utils import get_dsl_parser_by_version - - -@manager.route('/load', methods=['POST']) -def load_model(): - request_config = request.json - - if request_config.get('job_id'): - retcode, retmsg, data = model_utils.query_model_info(model_version=request_config['job_id'], role='guest') - if not data: - return get_json_result( - retcode=101, - retmsg=f"Model with version {request_config.get('job_id')} can not be found in database. " - "Please check if the model version is valid.", - ) - - model_info = data[0] - request_config['initiator'] = {} - request_config['initiator']['party_id'] = str(model_info.get('f_initiator_party_id')) - request_config['initiator']['role'] = model_info.get('f_initiator_role') - runtime_conf = model_info.get('f_runtime_conf', {}) if model_info.get('f_runtime_conf', {}) else model_info.get('f_train_runtime_conf', {}) - adapter = JobRuntimeConfigAdapter(runtime_conf) - job_parameters = adapter.get_common_parameters().to_dict() - request_config['job_parameters'] = job_parameters if job_parameters else model_info.get('f_train_runtime_conf', {}).get('job_parameters') - roles = runtime_conf.get('role') - request_config['role'] = roles if roles else model_info.get('f_train_runtime_conf', {}).get('role') - for key, value in request_config['role'].items(): - for i, v in enumerate(value): - value[i] = str(v) - request_config.pop('job_id') - - _job_id = job_utils.generate_job_id() - initiator_party_id = request_config['initiator']['party_id'] - initiator_role = request_config['initiator']['role'] - publish_model.generate_publish_model_info(request_config) - load_status = True - load_status_info = {} - load_status_msg = 'success' - load_status_info['detail'] = {} - if "federated_mode" not in request_config['job_parameters']: - if IS_STANDALONE: - request_config['job_parameters']["federated_mode"] = FederatedMode.SINGLE - else: - request_config['job_parameters']["federated_mode"] = FederatedMode.MULTIPLE - for role_name, role_partys in request_config.get("role").items(): - if role_name == 'arbiter': - continue - load_status_info[role_name] = load_status_info.get(role_name, {}) - load_status_info['detail'][role_name] = {} - for _party_id in role_partys: - request_config['local'] = {'role': role_name, 'party_id': _party_id} - try: - response = federated_api(job_id=_job_id, - method='POST', - endpoint='/model/load/do', - src_party_id=initiator_party_id, - dest_party_id=_party_id, - src_role = initiator_role, - json_body=request_config, - federated_mode=request_config['job_parameters']['federated_mode']) - load_status_info[role_name][_party_id] = response['retcode'] - detail = {_party_id: {}} - detail[_party_id]['retcode'] = response['retcode'] - detail[_party_id]['retmsg'] = response['retmsg'] - load_status_info['detail'][role_name].update(detail) - if response['retcode']: - load_status = False - load_status_msg = 'failed' - except Exception as e: - stat_logger.exception(e) - load_status = False - load_status_msg = 'failed' - load_status_info[role_name][_party_id] = 100 - return get_json_result(job_id=_job_id, retcode=(0 if load_status else 101), retmsg=load_status_msg, - data=load_status_info) - - -@manager.route('/migrate', methods=['POST']) -@validate_request("migrate_initiator", "role", "migrate_role", "model_id", - "model_version", "execute_party", "job_parameters") -def migrate_model_process(): - request_config = request.json - _job_id = job_utils.generate_job_id() - initiator_party_id = request_config['migrate_initiator']['party_id'] - initiator_role = request_config['migrate_initiator']['role'] - if not request_config.get("unify_model_version"): - request_config["unify_model_version"] = _job_id - migrate_status = True - migrate_status_info = {} - migrate_status_msg = 'success' - migrate_status_info['detail'] = {} - - try: - if migrate_model.compare_roles(request_config.get("migrate_role"), request_config.get("role")): - return get_json_result(retcode=100, - retmsg="The config of previous roles is the same with that of migrate roles. " - "There is no need to migrate model. Migration process aborting.") - except Exception as e: - return get_json_result(retcode=100, retmsg=str(e)) - - local_template = { - "role": "", - "party_id": "", - "migrate_party_id": "" - } - - res_dict = {} - - for role_name, role_partys in request_config.get("migrate_role").items(): - for offset, party_id in enumerate(role_partys): - local_res = deepcopy(local_template) - local_res["role"] = role_name - local_res["party_id"] = request_config.get("role").get(role_name)[offset] - local_res["migrate_party_id"] = party_id - if not res_dict.get(role_name): - res_dict[role_name] = {} - res_dict[role_name][local_res["party_id"]] = local_res - - for role_name, role_partys in request_config.get("execute_party").items(): - migrate_status_info[role_name] = migrate_status_info.get(role_name, {}) - migrate_status_info['detail'][role_name] = {} - for party_id in role_partys: - request_config["local"] = res_dict.get(role_name).get(party_id) - try: - response = federated_api(job_id=_job_id, - method='POST', - endpoint='/model/migrate/do', - src_party_id=initiator_party_id, - dest_party_id=party_id, - src_role=initiator_role, - json_body=request_config, - federated_mode=request_config['job_parameters']['federated_mode']) - migrate_status_info[role_name][party_id] = response['retcode'] - detail = {party_id: {}} - detail[party_id]['retcode'] = response['retcode'] - detail[party_id]['retmsg'] = response['retmsg'] - migrate_status_info['detail'][role_name].update(detail) - except Exception as e: - stat_logger.exception(e) - migrate_status = False - migrate_status_msg = 'failed' - migrate_status_info[role_name][party_id] = 100 - return get_json_result(job_id=_job_id, retcode=(0 if migrate_status else 101), - retmsg=migrate_status_msg, data=migrate_status_info) - - -@manager.route('/migrate/do', methods=['POST']) -def do_migrate_model(): - request_data = request.json - retcode, retmsg, data = migrate_model.migration(request_data) - return get_json_result(retcode=retcode, retmsg=retmsg, data=data) - - -@manager.route('/load/do', methods=['POST']) -def do_load_model(): - request_data = request.json - request_data['servings'] = RuntimeConfig.SERVICE_DB.get_urls('servings') - - role = request_data['local']['role'] - party_id = request_data['local']['party_id'] - model_id = request_data['job_parameters']['model_id'] - model_version = request_data['job_parameters']['model_version'] - - if ENABLE_MODEL_STORE: - sync_model = SyncModel( - role=role, party_id=party_id, - model_id=model_id, model_version=model_version, - ) - - if sync_model.remote_exists(): - sync_model.download(True) - - if not model_utils.check_if_deployed(role, party_id, model_id, model_version): - return get_json_result(retcode=100, - retmsg="Only deployed models could be used to execute process of loading. " - "Please deploy model before loading.") - - retcode, retmsg = publish_model.load_model(request_data) - try: - if not retcode: - with DB.connection_context(): - model = MLModel.get_or_none( - MLModel.f_role == role, - MLModel.f_party_id == party_id, - MLModel.f_model_id == model_id, - MLModel.f_model_version == model_version, - ) - if model: - model.f_loaded_times += 1 - model.save() - except Exception as modify_err: - stat_logger.exception(modify_err) - - return get_json_result(retcode=retcode, retmsg=retmsg) - - -@manager.route('/bind', methods=['POST']) -def bind_model_service(): - request_config = request.json - - if request_config.get('job_id'): - retcode, retmsg, data = model_utils.query_model_info(model_version=request_config['job_id'], role='guest') - if not data: - return get_json_result( - retcode=101, - retmsg=f"Model {request_config.get('job_id')} can not be found in database. " - "Please check if the model version is valid." - ) - - model_info = data[0] - request_config['initiator'] = {} - request_config['initiator']['party_id'] = str(model_info.get('f_initiator_party_id')) - request_config['initiator']['role'] = model_info.get('f_initiator_role') - - runtime_conf = model_info.get('f_runtime_conf', {}) if model_info.get('f_runtime_conf', {}) else model_info.get('f_train_runtime_conf', {}) - adapter = JobRuntimeConfigAdapter(runtime_conf) - job_parameters = adapter.get_common_parameters().to_dict() - request_config['job_parameters'] = job_parameters if job_parameters else model_info.get('f_train_runtime_conf', {}).get('job_parameters') - - roles = runtime_conf.get('role') - request_config['role'] = roles if roles else model_info.get('f_train_runtime_conf', {}).get('role') - - for key, value in request_config['role'].items(): - for i, v in enumerate(value): - value[i] = str(v) - request_config.pop('job_id') - - if not request_config.get('servings'): - # get my party all servings - request_config['servings'] = RuntimeConfig.SERVICE_DB.get_urls('servings') - service_id = request_config.get('service_id') - if not service_id: - return get_json_result(retcode=101, retmsg='no service id') - detect_utils.check_config(request_config, ['initiator', 'role', 'job_parameters']) - bind_status, retmsg = publish_model.bind_model_service(request_config) - - return get_json_result(retcode=bind_status, retmsg='service id is {}'.format(service_id) if not retmsg else retmsg) - - -@manager.route('/transfer', methods=['post']) -def transfer_model(): - party_model_id = request.json.get('namespace') - model_version = request.json.get('name') - if not party_model_id or not model_version: - return error_response(400, 'namespace and name are required') - model_data = publish_model.download_model(party_model_id, model_version) - if not model_data: - return error_response(404, 'model not found') - return get_json_result(data=model_data) - - -@manager.route('/transfer//', methods=['post']) -def download_model(party_model_id, model_version): - party_model_id = party_model_id.replace('~', '#') - model_data = publish_model.download_model(party_model_id, model_version) - if not model_data: - return error_response(404, 'model not found') - return get_json_result(data=model_data) - - -@manager.route('/', methods=['post', 'get']) -@validate_request("model_id", "model_version", "role", "party_id") -def operate_model(model_operation): - request_config = request.json or request.form.to_dict() - job_id = job_utils.generate_job_id() - - # TODO: export, import, store, restore should NOT be in the same function - if not ModelOperation.valid(model_operation): - raise Exception(f'Not supported model operation: "{model_operation}".') - model_operation = ModelOperation(model_operation) - - request_config['party_id'] = str(request_config['party_id']) - request_config['model_version'] = str(request_config['model_version']) - party_model_id = model_utils.gen_party_model_id( - request_config['model_id'], - request_config['role'], - request_config['party_id'], - ) - - if model_operation in [ModelOperation.EXPORT, ModelOperation.IMPORT]: - - if model_operation is ModelOperation.IMPORT: - file = request.files.get('file') - if not file: - return error_response(400, '`file` is required.') - - force_update = bool(int(request_config.get('force_update', 0))) - - if not force_update: - with DB.connection_context(): - if MLModel.get_or_none( - MLModel.f_role == request_config['role'], - MLModel.f_party_id == request_config['party_id'], - MLModel.f_model_id == request_config['model_id'], - MLModel.f_model_version == request_config['model_version'], - ): - return error_response(409, 'Model already exists.') - - filename = os.path.join(TEMP_DIRECTORY, uuid1().hex) - os.makedirs(os.path.dirname(filename), exist_ok=True) - - try: - file.save(filename) - except Exception as e: - try: - filename.unlink() - except FileNotFoundError: - pass - - return error_response(500, f'Save file error: {e}') - - model = PipelinedModel(party_model_id, request_config['model_version']) - model.unpack_model(filename, force_update, request_config.get('hash')) - - pipeline = model.read_pipeline_model() - train_runtime_conf = json_loads(pipeline.train_runtime_conf) - - for _party_id in train_runtime_conf['role'].get(request_config['role'], []): - if request_config['party_id'] == str(_party_id): - break - else: - shutil.rmtree(model.model_path, ignore_errors=True) - return error_response( - 400, - f'Party id "{request_config["party_id"]}" is not in role "{request_config["role"]}", ' - f'please check if the party id and role is valid.', - ) - - model.pipelined_component.save_define_meta_from_file_to_db(force_update) - - if ENABLE_MODEL_STORE: - query = model.pipelined_component.get_define_meta_from_db( - PipelineComponentMeta.f_component_name != PIPELINE_COMPONENT_NAME, - ) - for row in query: - sync_component = SyncComponent( - role=request_config['role'], party_id=request_config['party_id'], - model_id=request_config['model_id'], model_version=request_config['model_version'], - component_name=row.f_component_name, - ) - sync_component.upload() - - pipeline.model_id = request_config['model_id'] - pipeline.model_version = request_config['model_version'] - - train_runtime_conf = JobRuntimeConfigAdapter( - train_runtime_conf, - ).update_model_id_version( - model_id=request_config['model_id'], - model_version=request_config['model_version'], - ) - - if compare_version(pipeline.fate_version, '1.5.0') == 'gt': - runtime_conf_on_party = json_loads(pipeline.runtime_conf_on_party) - runtime_conf_on_party['job_parameters']['model_id'] = request_config['model_id'] - runtime_conf_on_party['job_parameters']['model_version'] = request_config['model_version'] - - # fix migrate bug between 1.5.x and 1.8.x - if compare_version(pipeline.fate_version, '1.9.0') == 'lt': - pipeline.roles = json_dumps(train_runtime_conf['role'], byte=True) - - runtime_conf_on_party['role'] = train_runtime_conf['role'] - runtime_conf_on_party['initiator'] = train_runtime_conf['initiator'] - - pipeline.runtime_conf_on_party = json_dumps(runtime_conf_on_party, byte=True) - - model.save_pipeline_model(pipeline, False) - - model_info = model_utils.gather_model_info_data(model) - model_info['f_role'] = request_config['role'] - model_info['f_party_id'] = request_config['party_id'] - model_info['f_job_id'] = job_id - model_info['f_imported'] = 1 - model_utils.save_model_info(model_info) - - return get_json_result(data={ - 'job_id': job_id, - 'role': request_config['role'], - 'party_id': request_config['party_id'], - 'model_id': request_config['model_id'], - 'model_version': request_config['model_version'], - }) - - # export - else: - if ENABLE_MODEL_STORE: - sync_model = SyncModel( - role=request_config['role'], party_id=request_config['party_id'], - model_id=request_config['model_id'], model_version=request_config['model_version'], - ) - if sync_model.remote_exists(): - sync_model.download(True) - - model = PipelinedModel(party_model_id, request_config["model_version"]) - if not model.exists(): - return error_response(404, f"Model {party_model_id} {request_config['model_version']} does not exist.") - - model.packaging_model() - return send_file( - model.archive_model_file_path, - as_attachment=True, - attachment_filename=os.path.basename(model.archive_model_file_path), - ) - - # store and restore - else: - request_config['model_id'] = party_model_id - - job_dsl, job_runtime_conf = gen_model_operation_job_config(request_config, model_operation) - submit_result = DAGScheduler.submit(JobConfigurationBase(**{'dsl': job_dsl, 'runtime_conf': job_runtime_conf}), job_id=job_id) - - return get_json_result(job_id=job_id, data=submit_result) - - -@manager.route('/model_tag/', methods=['POST']) -@DB.connection_context() -def tag_model(operation): - if operation not in ['retrieve', 'create', 'remove']: - return get_json_result(retcode=100, retmsg="'{}' is not currently supported.".format(operation)) - - request_data = request.json - model = MLModel.get_or_none(MLModel.f_model_version == request_data.get("job_id")) - if not model: - raise Exception("Can not found model by job id: '{}'.".format(request_data.get("job_id"))) - - if operation == 'retrieve': - res = {'tags': []} - tags = (Tag.select().join(ModelTag, on=ModelTag.f_t_id == Tag.f_id).where(ModelTag.f_m_id == model.f_model_version)) - for tag in tags: - res['tags'].append({'name': tag.f_name, 'description': tag.f_desc}) - res['count'] = tags.count() - return get_json_result(data=res) - elif operation == 'remove': - tag = Tag.get_or_none(Tag.f_name == request_data.get('tag_name')) - if not tag: - raise Exception("Can not found '{}' tag.".format(request_data.get('tag_name'))) - tags = (Tag.select().join(ModelTag, on=ModelTag.f_t_id == Tag.f_id).where(ModelTag.f_m_id == model.f_model_version)) - if tag.f_name not in [t.f_name for t in tags]: - raise Exception("Model {} {} does not have tag '{}'.".format(model.f_model_id, - model.f_model_version, - tag.f_name)) - delete_query = ModelTag.delete().where(ModelTag.f_m_id == model.f_model_version, ModelTag.f_t_id == tag.f_id) - delete_query.execute() - return get_json_result(retmsg="'{}' tag has been removed from tag list of model {} {}.".format(request_data.get('tag_name'), - model.f_model_id, - model.f_model_version)) - else: - if not str(request_data.get('tag_name')): - raise Exception("Tag name should not be an empty string.") - tag = Tag.get_or_none(Tag.f_name == request_data.get('tag_name')) - if not tag: - tag = Tag() - tag.f_name = request_data.get('tag_name') - tag.save(force_insert=True) - else: - tags = (Tag.select().join(ModelTag, on=ModelTag.f_t_id == Tag.f_id).where(ModelTag.f_m_id == model.f_model_version)) - if tag.f_name in [t.f_name for t in tags]: - raise Exception("Model {} {} already been tagged as tag '{}'.".format(model.f_model_id, - model.f_model_version, - tag.f_name)) - ModelTag.create(f_t_id=tag.f_id, f_m_id=model.f_model_version) - return get_json_result(retmsg="Adding {} tag for model with job id: {} successfully.".format(request_data.get('tag_name'), - request_data.get('job_id'))) - - -@manager.route('/tag/', methods=['POST']) -@DB.connection_context() -def operate_tag(tag_operation): - request_data = request.json - if not TagOperation.valid(tag_operation): - raise Exception('The {} operation is not currently supported.'.format(tag_operation)) - - tag_name = request_data.get('tag_name') - tag_desc = request_data.get('tag_desc') - tag_operation = TagOperation(tag_operation) - if tag_operation is TagOperation.CREATE: - try: - if not tag_name: - return get_json_result(retcode=100, retmsg="'{}' tag created failed. Please input a valid tag name.".format(tag_name)) - else: - Tag.create(f_name=tag_name, f_desc=tag_desc) - except peewee.IntegrityError: - raise Exception("'{}' has already exists in database.".format(tag_name)) - else: - return get_json_result(retmsg="'{}' tag has been created successfully.".format(tag_name)) - - elif tag_operation is TagOperation.LIST: - tags = Tag.select() - limit = request_data.get('limit') - res = {"tags": []} - - if limit > len(tags): - count = len(tags) - else: - count = limit - for tag in tags[:count]: - res['tags'].append({'name': tag.f_name, 'description': tag.f_desc, - 'model_count': ModelTag.filter(ModelTag.f_t_id == tag.f_id).count()}) - return get_json_result(data=res) - - else: - if not (tag_operation is TagOperation.RETRIEVE and not request_data.get('with_model')): - try: - tag = Tag.get(Tag.f_name == tag_name) - except peewee.DoesNotExist: - raise Exception("Can not found '{}' tag.".format(tag_name)) - - if tag_operation is TagOperation.RETRIEVE: - if request_data.get('with_model', False): - res = {'models': []} - models = (MLModel.select().join(ModelTag, on=ModelTag.f_m_id == MLModel.f_model_version).where(ModelTag.f_t_id == tag.f_id)) - for model in models: - res["models"].append({ - "model_id": model.f_model_id, - "model_version": model.f_model_version, - "model_size": model.f_size, - "role": model.f_role, - "party_id": model.f_party_id - }) - res["count"] = models.count() - return get_json_result(data=res) - else: - tags = Tag.filter(Tag.f_name.contains(tag_name)) - if not tags: - return get_json_result(retcode=100, retmsg="No tags found.") - res = {'tags': []} - for tag in tags: - res['tags'].append({'name': tag.f_name, 'description': tag.f_desc}) - return get_json_result(data=res) - - elif tag_operation is TagOperation.UPDATE: - new_tag_name = request_data.get('new_tag_name', None) - new_tag_desc = request_data.get('new_tag_desc', None) - if (tag.f_name == new_tag_name) and (tag.f_desc == new_tag_desc): - return get_json_result(100, "Nothing to be updated.") - else: - if request_data.get('new_tag_name'): - if not Tag.get_or_none(Tag.f_name == new_tag_name): - tag.f_name = new_tag_name - else: - return get_json_result(100, retmsg="'{}' tag already exists.".format(new_tag_name)) - - tag.f_desc = new_tag_desc - tag.save() - return get_json_result(retmsg="Infomation of '{}' tag has been updated successfully.".format(tag_name)) - - else: - delete_query = ModelTag.delete().where(ModelTag.f_t_id == tag.f_id) - delete_query.execute() - Tag.delete_instance(tag) - return get_json_result(retmsg="'{}' tag has been deleted successfully.".format(tag_name)) - - -def gen_model_operation_job_config(config_data: dict, model_operation: ModelOperation): - if model_operation not in {ModelOperation.STORE, ModelOperation.RESTORE}: - raise Exception("Can not support this model operation: {}".format(model_operation)) - - component_name = f"{str(model_operation).replace('.', '_').lower()}_0" - - job_dsl = { - "components": { - component_name: { - "module": "Model{}".format(model_operation.value.capitalize()), - }, - }, - } - - job_runtime_conf = job_utils.runtime_conf_basic(True) - - component_parameters = { - "model_id": config_data["model_id"], - "model_version": config_data["model_version"], - "store_address": ServerRegistry.MODEL_STORE_ADDRESS, - } - if model_operation == ModelOperation.STORE: - component_parameters["force_update"] = config_data.get("force_update", False) - elif model_operation == ModelOperation.RESTORE: - component_parameters["hash_"] = config_data.get("sha256", None) - - job_runtime_conf["component_parameters"]["role"] = { - "local": { - "0": { - component_name: component_parameters, - }, - }, - } - - return job_dsl, job_runtime_conf - - -@manager.route('/query', methods=['POST']) -def query_model(): - request_data = request.json or request.form.to_dict() or {} - - retcode, retmsg, data = model_utils.query_model_info(**request_data) - return get_json_result(retcode=retcode, retmsg=retmsg, data=data) - - -@manager.route('/query/detail', methods=['POST']) -def query_model_detail(): - retcode, retmsg, data = model_utils.query_model_detail(**request.json) - return get_json_result(retcode=retcode, retmsg=retmsg, data=data) - - -@manager.route('/deploy', methods=['POST']) -@validate_request('model_id', 'model_version') -def deploy(): - request_data = request.json - - model_id = request_data['model_id'] - model_version = request_data['model_version'] - - if not isinstance(request_data.get('components_checkpoint'), dict): - request_data['components_checkpoint'] = {} - - retcode, retmsg, data = model_utils.query_model_info(model_id=model_id, model_version=model_version) - if not data: - return error_response( - 404, - 'Deploy model failed. ' - f'Model {model_id} {model_version} not found.' - ) - - for model_info in data: - version_check = compare_version(model_info.get('f_fate_version'), '1.5.0') - if version_check == 'lt': - continue - - initiator_role = (model_info['f_initiator_role'] if model_info.get('f_initiator_role') - else model_info.get('f_train_runtime_conf', {}).get('initiator', {}).get('role', '')) - initiator_party_id = (model_info['f_initiator_party_id'] if model_info.get('f_initiator_party_id') - else model_info.get('f_train_runtime_conf', {}).get('initiator', {}).get('party_id', '')) - - if model_info['f_role'] == initiator_role and str(model_info['f_party_id']) == str(initiator_party_id): - break - else: - return error_response( - 404, - 'Deploy model failed. ' - 'Cannot found model of initiator role or the fate version of model is older than 1.5.0', - ) - - roles = ( - data[0].get('f_roles') or - data[0].get('f_train_runtime_conf', {}).get('role') or - data[0].get('f_runtime_conf', {}).get('role') - ) - if not roles: - return error_response( - 404, - 'Deploy model failed. ' - 'Cannot found roles of model.' - ) - - # distribute federated deploy task - _job_id = job_utils.generate_job_id() - request_data['child_model_version'] = _job_id - request_data['initiator'] = { - 'role': initiator_role, - 'party_id': initiator_party_id, - } - - deploy_status = True - deploy_status_info = { - 'detail': {}, - 'model_id': model_id, - 'model_version': _job_id, - } - - for role_name, role_partys in roles.items(): - if role_name not in {'arbiter', 'host', 'guest'}: - continue - - if role_name not in deploy_status_info: - deploy_status_info[role_name] = {} - if role_name not in deploy_status_info['detail']: - deploy_status_info['detail'][role_name] = {} - - for _party_id in role_partys: - request_data['local'] = { - 'role': role_name, - 'party_id': _party_id, - } - - try: - response = federated_api( - job_id=_job_id, - method='POST', - endpoint='/model/deploy/do', - src_party_id=initiator_party_id, - dest_party_id=_party_id, - src_role=initiator_role, - json_body=request_data, - federated_mode=FederatedMode.MULTIPLE if not IS_STANDALONE else FederatedMode.SINGLE - ) - if response['retcode']: - deploy_status = False - - deploy_status_info[role_name][_party_id] = response['retcode'] - deploy_status_info['detail'][role_name][_party_id] = { - 'retcode': response['retcode'], - 'retmsg': response['retmsg'], - } - except Exception as e: - deploy_status = False - - deploy_status_info[role_name][_party_id] = 100 - deploy_status_info['detail'][role_name][_party_id] = { - 'retcode': 100, - 'retmsg': 'request failed', - } - - stat_logger.exception(e) - - return get_json_result( - 0 if deploy_status else 101, - 'success' if deploy_status else 'failed', - deploy_status_info, - ) - - -@manager.route('/deploy/do', methods=['POST']) -def do_deploy(): - retcode, retmsg = deploy_model.deploy(request.json) - - return get_json_result(retcode=retcode, retmsg=retmsg) - - -def get_dsl_and_conf(): - request_data = request.json or request.form.to_dict() or {} - request_data['query_filters'] = [ - 'model_id', - 'model_version', - 'role', - 'party_id', - 'train_runtime_conf', - 'inference_dsl', - ] - - retcode, retmsg, data = model_utils.query_model_info(**request_data) - - if not data: - abort(error_response( - 210, - 'No model found, ' - 'please check if arguments are specified correctly.', - )) - - for _data in data: - if _data.get('f_role') in {'guest', 'host'}: - data = _data - break - else: - abort(error_response( - 210, - 'Cannot found guest or host model, ' - 'please get predict dsl on guest or host.', - )) - - return request_data, data - - -@manager.route('/get/predict/dsl', methods=['POST']) -def get_predict_dsl(): - request_data, data = get_dsl_and_conf() - - if request_data.get('filename'): - return send_file_in_mem(data['f_inference_dsl'], request_data['filename']) - - return get_json_result(data=data['f_inference_dsl']) - - -@manager.route('/get/predict/conf', methods=['POST']) -def get_predict_conf(): - request_data, data = get_dsl_and_conf() - - parser = get_dsl_parser_by_version(data['f_train_runtime_conf'].get('dsl_version', 1)) - conf = parser.generate_predict_conf_template( - data['f_inference_dsl'], data['f_train_runtime_conf'], - data['f_model_id'], data['f_model_version'], - ) - - if request_data.get('filename'): - return send_file_in_mem(conf, request_data['filename']) - - return get_json_result(data=conf) - - -@manager.route('/archive/packaging', methods=['POST']) -@validate_request('party_model_id', 'model_version') -def packaging_model(): - request_data = request.json or request.form.to_dict() - - if ENABLE_MODEL_STORE: - sync_model = SyncModel( - party_model_id=request_data['party_model_id'], - model_version=request_data['model_version'], - ) - if sync_model.remote_exists(): - sync_model.download(True) - - model = PipelinedModel( - model_id=request_data['party_model_id'], - model_version=request_data['model_version'], - ) - - if not model.exists(): - return error_response(404, 'Model not found.') - - hash_ = model.packaging_model() - - return get_json_result(data={ - 'party_model_id': model.party_model_id, - 'model_version': model.model_version, - 'path': model.archive_model_file_path, - 'hash': hash_, - }) - - -@manager.route('/service/register', methods=['POST']) -@validate_request('party_model_id', 'model_version') -def register_service(): - request_data = request.json or request.form.to_dict() - - RuntimeConfig.SERVICE_DB.register_model( - party_model_id=request_data['party_model_id'], - model_version=request_data['model_version'], - ) - - return get_json_result(data={ - 'party_model_id': request_data['party_model_id'], - 'model_version': request_data['model_version'], - }) - - -@manager.route('/homo/convert', methods=['POST']) -@validate_request("model_id", "model_version", "role", "party_id") -def homo_convert(): - request_data = request.json or request.form.to_dict() - retcode, retmsg, res_data = publish_model.convert_homo_model(request_data) - - return get_json_result(retcode=retcode, retmsg=retmsg, data=res_data) - - -@manager.route('/homo/deploy', methods=['POST']) -@validate_request("service_id", "model_id", "model_version", "role", "party_id", - "component_name", "deployment_type", "deployment_parameters") -def homo_deploy(): - request_data = request.json or request.form.to_dict() - retcode, retmsg, res_data = publish_model.deploy_homo_model(request_data) - - return get_json_result(retcode=retcode, retmsg=retmsg, data=res_data) diff --git a/python/fate_flow/errors/general_error.py b/python/fate_flow/apps/partner/__init__.py similarity index 88% rename from python/fate_flow/errors/general_error.py rename to python/fate_flow/apps/partner/__init__.py index 9c4d30e34..b4ade8a27 100644 --- a/python/fate_flow/errors/general_error.py +++ b/python/fate_flow/apps/partner/__init__.py @@ -13,9 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -class ParameterError(Exception): - pass -class PassError(Exception): - pass +# Define federated scheduling api diff --git a/python/fate_flow/apps/partner/partner_app.py b/python/fate_flow/apps/partner/partner_app.py new file mode 100644 index 000000000..999fcfc2d --- /dev/null +++ b/python/fate_flow/apps/partner/partner_app.py @@ -0,0 +1,272 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from webargs import fields + +from fate_flow.controller.job import JobController +from fate_flow.controller.task import TaskController +from fate_flow.entity.types import TaskStatus +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import CreateJobFailed, UpdateJobFailed, KillFailed, JobResourceException,\ + NoFoundTask, StartTaskFailed, UpdateTaskFailed, KillTaskFailed, TaskResourceException +from fate_flow.manager.service.resource_manager import ResourceManager +from fate_flow.manager.operation.job_saver import JobSaver +from fate_flow.utils.api_utils import API, stat_logger +from fate_flow.utils.permission_utils import create_job_request_check +from fate_flow.utils.wraps_utils import task_request_proxy + +page_name = 'partner' + + +@manager.route('/job/create', methods=['POST']) +@API.Input.json(dag=fields.Dict(required=True)) +@API.Input.json(schema_version=fields.String(required=True)) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@create_job_request_check +def partner_create_job(dag, schema_version, job_id, role, party_id): + try: + JobController.create_job(dag, schema_version, job_id, role, party_id) + return API.Output.json() + except Exception as e: + stat_logger.exception(e) + return API.Output.fate_flow_exception(CreateJobFailed(detail=str(e))) + + +@manager.route('/job/start', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(extra_info=fields.Dict(required=False)) +def start_job(job_id, role, party_id, extra_info=None): + JobController.start_job(job_id=job_id, role=role, party_id=party_id, extra_info=extra_info) + return API.Output.json() + + +@manager.route('/job/status/update', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(status=fields.String(required=True)) +def partner_job_status_update(job_id, role, party_id, status): + job_info = { + "job_id": job_id, + "role": role, + "party_id": party_id, + "status": status + } + if JobController.update_job_status(job_info=job_info): + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success') + else: + return API.Output.fate_flow_exception(UpdateJobFailed( + job_id=job_id, role=role, party_id=party_id, status=status + )) + + +@manager.route('/job/update', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(progress=fields.Float()) +def partner_job_update(job_id, role, party_id, progress): + job_info = { + "job_id": job_id, + "role": role, + "party_id": party_id + } + if progress: + job_info.update({"progress": progress}) + if JobController.update_job(job_info=job_info): + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success') + else: + return API.Output.fate_flow_exception(UpdateJobFailed(**job_info)) + + +@manager.route('/job/resource/apply', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +def apply_resource(job_id, role, party_id): + status = ResourceManager.apply_for_job_resource(job_id, role, party_id) + if status: + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success') + else: + return API.Output.fate_flow_exception(JobResourceException( + job_id=job_id, role=role, party_id=party_id, + operation_type="apply" + )) + + +@manager.route('/job/resource/return', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +def return_resource(job_id, role, party_id): + status = ResourceManager.return_job_resource(job_id=job_id, role=role, party_id=party_id) + if status: + return API.Output.json(ReturnCode.Base.SUCCESS, message='success') + else: + return API.Output.fate_flow_exception(JobResourceException( + job_id=job_id, role=role, party_id=party_id, + operation_type="return" + )) + + +@manager.route('/job/stop', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +def stop_job(job_id, role, party_id): + kill_status, kill_details = JobController.stop_jobs(job_id=job_id, role=role, party_id=party_id) + if kill_status: + return API.Output.json() + return API.Output.fate_flow_exception(KillFailed(detail=kill_details)) + + +@manager.route('/task/resource/apply', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) +def apply_task_resource(job_id, role, party_id, task_id, task_version): + status = ResourceManager.apply_for_task_resource(job_id=job_id, role=role, party_id=party_id, + task_id=task_id, task_version=task_version) + if status: + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success') + else: + return API.Output.fate_flow_exception(TaskResourceException( + job_id=job_id, role=role, party_id=party_id, + task_id=task_id, task_version=task_version, operation_type="apply" + )) + + +@manager.route('/task/resource/return', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) +def return_task_resource(job_id, role, party_id, task_id, task_version): + status = ResourceManager.return_task_resource(job_id=job_id, role=role, party_id=party_id, + task_id=task_id, task_version=task_version) + if status: + return API.Output.json(ReturnCode.Base.SUCCESS, message='success') + else: + return API.Output.fate_flow_exception(TaskResourceException( + job_id=job_id, role=role, party_id=party_id, task_id=task_id, + task_version=task_version, operation_type="return" + )) + + +@manager.route('/task/start', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) +@task_request_proxy(filter_local=True) +def start_task(job_id, role, party_id, task_id, task_version): + task = JobSaver.query_task(task_id=task_id, task_version=task_version, role=role, party_id=party_id)[0] + if not task: + return API.Output.fate_flow_exception( + NoFoundTask(job_id=job_id, role=role, party_id=party_id, task_id=task_id, task_version=task_version) + ) + if TaskController.start_task(task): + return API.Output.json(code=ReturnCode.Base.SUCCESS, message='success') + else: + return API.Output.fate_flow_exception(StartTaskFailed( + job_id=job_id, role=role, party_id=party_id, + task_id=task_id, task_version=task_version + )) + + +@manager.route('/task/collect', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) +def collect_task(job_id, role, party_id, task_id, task_version): + task_info = TaskController.collect_task(job_id=job_id, task_id=task_id, task_version=task_version, role=role, + party_id=party_id) + if task_info: + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=task_info) + else: + return API.Output.fate_flow_exception(NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_id=task_id, task_version=task_version)) + + +@manager.route('/task/status/update', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) +@API.Input.json(status=fields.String(required=True)) +def task_status_update(job_id, role, party_id, task_id, task_version, status): + task_info = {} + task_info.update({ + "job_id": job_id, + "task_id": task_id, + "task_version": task_version, + "role": role, + "party_id": party_id, + "status": status + }) + if TaskController.update_task_status(task_info=task_info): + return API.Output.json() + else: + return API.Output.fate_flow_exception(UpdateTaskFailed( + job_id=job_id, role=role, party_id=party_id, + task_id=task_id, task_version=task_version, status=status) + ) + + +@manager.route('/task/stop', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) +@API.Input.json(status=fields.String()) +@task_request_proxy() +def stop_task(job_id, role, party_id, task_id, task_version, status=None): + if not status: + status = TaskStatus.FAILED + tasks = JobSaver.query_task(job_id=job_id, task_id=task_id, task_version=task_version, role=role, party_id=party_id) + kill_status = True + for task in tasks: + kill_status = kill_status & TaskController.stop_task(task=task, stop_status=status) + if kill_status: + return API.Output.json() + else: + return API.Output.fate_flow_exception(KillTaskFailed(job_id=job_id, role=role, party_id=party_id, + task_id=task_id, task_version=task_version)) + + +@manager.route('/task/rerun', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) +@API.Input.json(new_version=fields.Integer(required=True)) +def rerun_task(job_id, role, party_id, task_id, task_version, new_version): + tasks = JobSaver.query_task(job_id=job_id, task_id=task_id, role=role, party_id=party_id) + if tasks: + TaskController.create_new_version_task(task=tasks[0], new_version=new_version) + return API.Output.json() diff --git a/python/fate_flow/apps/permission_app.py b/python/fate_flow/apps/permission_app.py deleted file mode 100644 index 9fdc1acbd..000000000 --- a/python/fate_flow/apps/permission_app.py +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from flask import request - -from fate_flow.controller.permission_controller import PermissionController -from fate_flow.entity.permission_parameters import PermissionParameters -from fate_flow.utils import api_utils -from fate_flow.utils.api_utils import get_json_result - - -@manager.route('/grant', methods=['post']) -@api_utils.validate_request('party_id') -def grant_permission(): - parameters = PermissionParameters(**request.json) - PermissionController(parameters.party_id).grant_or_delete(parameters) - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('/delete', methods=['post']) -@api_utils.validate_request('party_id') -def delete_permission(): - parameters = PermissionParameters(is_delete=True, **request.json) - PermissionController(parameters.party_id).grant_or_delete(parameters) - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('/query', methods=['post']) -@api_utils.validate_request('party_id') -def query_privilege(): - parameters = PermissionParameters(**request.json) - data = PermissionController(parameters.party_id).query() - return get_json_result(retcode=0, retmsg='success', data=data) diff --git a/python/fate_flow/apps/provider_app.py b/python/fate_flow/apps/provider_app.py deleted file mode 100644 index 18920a7f5..000000000 --- a/python/fate_flow/apps/provider_app.py +++ /dev/null @@ -1,91 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import copy -from pathlib import Path - -from flask import request - -from fate_flow.db.component_registry import ComponentRegistry -from fate_flow.entity import ComponentProvider, RetCode -from fate_flow.entity.types import WorkerName -from fate_flow.manager.worker_manager import WorkerManager -from fate_flow.scheduler.cluster_scheduler import ClusterScheduler -from fate_flow.utils.api_utils import ( - error_response, get_json_result, - validate_request, -) - - -@manager.route('/update', methods=['POST']) -def provider_update(): - request_data = request.json - ComponentRegistry.load() - if ComponentRegistry.get_providers().get(request_data.get("name"), {}).get(request_data.get("version"), None) is None: - return get_json_result(retcode=RetCode.OPERATING_ERROR, retmsg=f"not load into memory") - return get_json_result() - - -@manager.route('/register', methods=['POST']) -@validate_request("name", "version", "path") -def register(): - info = request.json or request.form.to_dict() - - path = Path(info["path"]).absolute() - if not path.is_dir(): - return error_response(400, f"path '{path}' is not a directory") - if set(path.parent.iterdir()) - {path, (path.parent / "__init__.py")}: - return error_response(400, f"there are other directories or files in '{path.parent}' besides '{path.name}' and '__init__.py'") - - provider = ComponentProvider(name=info["name"], - version=info["version"], - path=info["path"], - class_path=info.get("class_path", ComponentRegistry.get_default_class_path())) - code, std = WorkerManager.start_general_worker(worker_name=WorkerName.PROVIDER_REGISTRAR, provider=provider) - - if code != 0: - return get_json_result(retcode=RetCode.OPERATING_ERROR, retmsg=f"register failed:\n{std}") - - federated_response = ClusterScheduler.cluster_command( - "/provider/update", - { - "name": info["name"], - "version": info["version"], - }, - ) - return get_json_result(data=federated_response) - - -@manager.route('/registry/get', methods=['POST']) -def get_registry(): - return get_json_result(data=ComponentRegistry.REGISTRY) - - -@manager.route('/get', methods=['POST']) -def get_providers(): - providers = ComponentRegistry.get_providers() - result = {} - for name, group_detail in providers.items(): - result[name] = {} - for version, detail in group_detail.items(): - result[name][version] = copy.deepcopy(detail) - if "components" in detail: - result[name][version]["components"] = set([c.lower() for c in detail["components"].keys()]) - return get_json_result(data=result) - - -@manager.route('//get', methods=['POST']) -def get_provider(provider_name): - return get_json_result(data=ComponentRegistry.get_providers().get(provider_name)) diff --git a/python/fate_flow/apps/proxy_app.py b/python/fate_flow/apps/proxy_app.py deleted file mode 100644 index 3d31369dc..000000000 --- a/python/fate_flow/apps/proxy_app.py +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from flask import request -from flask.json import jsonify - -from fate_arch.common import FederatedMode - -from fate_flow.utils.api_utils import federated_api, forward_api, proxy_api - - -page_name = 'forward' - - -@manager.route('/', methods=['POST']) -def start_proxy(role): - _job_id = f'{role}_forward' - request_config = request.json or request.form.to_dict() - - if request_config.get('header') and request_config.get('body'): - request_config['header'] = { - **request.headers, - **{ - k.replace('_', '-').upper(): v - for k, v in request_config['header'].items() - }, - } - else: - request_config = { - 'header': request.headers, - 'body': request_config, - } - - response = ( - proxy_api(role, _job_id, request_config) if role == 'marketplace' - else federated_api( - _job_id, 'POST', f'/forward/{role}/do', - request_config['header'].get('SRC-PARTY-ID'), - request_config['header'].get('DEST-PARTY-ID'), - '', request_config, FederatedMode.MULTIPLE, - ) - ) - return jsonify(response) - - -@manager.route('//do', methods=['POST']) -def start_forward(role): - request_config = request.json or request.form.to_dict() - return jsonify(forward_api(role, request_config)) diff --git a/python/fate_flow/apps/resource_app.py b/python/fate_flow/apps/resource_app.py deleted file mode 100644 index ebde69719..000000000 --- a/python/fate_flow/apps/resource_app.py +++ /dev/null @@ -1,36 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from flask import request - -from fate_flow.manager.resource_manager import ResourceManager -from fate_flow.utils.api_utils import get_json_result -from fate_flow.utils.api_utils import validate_request - - -@manager.route('/query', methods=['post']) -def query_resource(): - use_resource_job, computing_engine_resource = ResourceManager.query_resource(**request.json) - return get_json_result(retcode=0, retmsg='success', data={"use_resource_job": use_resource_job, - "computing_engine_resource": computing_engine_resource}) - - -@manager.route('/return', methods=['post']) -@validate_request('job_id') -def return_resource(): - status = ResourceManager.return_resource(job_id=request.json.get("job_id")) - return get_json_result(data=status) - - diff --git a/python/fate_flow/tests/__init__.py b/python/fate_flow/apps/scheduler/__init__.py similarity index 99% rename from python/fate_flow/tests/__init__.py rename to python/fate_flow/apps/scheduler/__init__.py index 878d3a9c5..ae946a49c 100644 --- a/python/fate_flow/tests/__init__.py +++ b/python/fate_flow/apps/scheduler/__init__.py @@ -12,4 +12,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# diff --git a/python/fate_flow/apps/scheduler/scheduler_app.py b/python/fate_flow/apps/scheduler/scheduler_app.py new file mode 100644 index 000000000..d9c69991d --- /dev/null +++ b/python/fate_flow/apps/scheduler/scheduler_app.py @@ -0,0 +1,71 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from webargs import fields + +from fate_flow.entity.spec.dag import DAGSchema +from fate_flow.errors.server_error import UpdateTaskFailed +from fate_flow.manager.operation.job_saver import ScheduleJobSaver +from fate_flow.scheduler.scheduler import DAGScheduler +from fate_flow.utils.api_utils import API + +page_name = 'scheduler' + + +@manager.route('/job/create', methods=['POST']) +@API.Input.json(dag=fields.Dict(required=True)) +@API.Input.json(schema_version=fields.String(required=True)) +def create_job(dag, schema_version): + dag = DAGSchema(dag=dag, schema_version=schema_version) + submit_result = DAGScheduler.create_all_job(dag.dict()) + return API.Output.json(**submit_result) + + +@manager.route('/task/report', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(role=fields.String(required=True)) +@API.Input.json(party_id=fields.String(required=True)) +@API.Input.json(task_id=fields.String(required=True)) +@API.Input.json(task_version=fields.Integer(required=True)) +@API.Input.json(status=fields.String(required=False)) +def report_task(job_id, role, party_id, task_id, task_version, status=None): + status = ScheduleJobSaver.update_task_status(task_info={ + "job_id": job_id, + "role": role, + "party_id": party_id, + "task_id": task_id, + "task_version": task_version, + "status": status + }) + if status: + return API.Output.json() + return API.Output.fate_flow_exception(UpdateTaskFailed( + job_id=job_id, role=role, party_id=party_id, + task_id=task_id, task_version=task_version, status=status) + ) + + +@manager.route('/job/stop', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +@API.Input.json(stop_status=fields.String(required=False)) +def stop_job(job_id, stop_status=None): + retcode, retmsg = DAGScheduler.stop_job(job_id, stop_status) + return API.Output.json(code=retcode, message=retmsg) + + +@manager.route('/job/rerun', methods=['POST']) +@API.Input.json(job_id=fields.String(required=True)) +def rerun_job(job_id): + DAGScheduler.rerun_job(job_id=job_id, auto=False) + return API.Output.json() diff --git a/python/fate_flow/apps/server_app.py b/python/fate_flow/apps/server_app.py deleted file mode 100644 index a79ca301f..000000000 --- a/python/fate_flow/apps/server_app.py +++ /dev/null @@ -1,74 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from flask import request - -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.utils.api_utils import get_json_result -from fate_flow.settings import API_VERSION -from fate_flow.db.service_registry import ServerRegistry -from fate_flow.db.config_manager import ConfigManager -from fate_flow.db.job_default_config import JobDefaultConfig -from fate_flow.entity import RetCode - - -@manager.route('/fateflow/info', methods=['POST']) -def fate_flow_server_info(): - data = RuntimeConfig.SERVICE_DB.get_servers() - return get_json_result(data=data) - - -@manager.route('/version/get', methods=['POST']) -def get_fate_version_info(): - module = (request.json or {}).get('module', None) - if module: - version = {module: RuntimeConfig.get_env(module)} - else: - version = RuntimeConfig.get_all_env() - version["API"] = API_VERSION - return get_json_result(data=version) - - -@manager.route('/get', methods=['POST']) -@manager.route('/service/get', methods=['POST']) -def get_server_registry(): - return get_json_result(data=ServerRegistry.get_all()) - - -@manager.route('//register', methods=['POST']) -@manager.route('/service//register', methods=['POST']) -def register_server(server_name: str): - ServerRegistry.register(server_name.upper(), request.json) - if ServerRegistry.get(server_name.upper()) is not None: - return get_json_result() - else: - return get_json_result(retcode=RetCode.OPERATING_ERROR) - - -@manager.route('//get', methods=['POST']) -@manager.route('/service//get', methods=['POST']) -def get_server(server_name: str): - return get_json_result(data=ServerRegistry.get(server_name.upper())) - - -@manager.route('/reload', methods=['POST']) -def reload(): - config = ConfigManager.load() - return get_json_result(data=config) - - -@manager.route('/config/job/default', methods=['POST']) -def job_default_config(): - return get_json_result(data=JobDefaultConfig.get_all()) diff --git a/python/fate_flow/apps/service_app.py b/python/fate_flow/apps/service_app.py deleted file mode 100644 index 0cda73b17..000000000 --- a/python/fate_flow/apps/service_app.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from flask import request - -from fate_flow.db.service_registry import ServiceRegistry, ServerRegistry -from fate_flow.utils.api_utils import get_json_result, validate_request - - -@manager.route("/registry", methods=['POST']) -def create_service(): - service_info = request.json - # compatibility - update_server = {} - if "server_name" not in service_info: - update_server = ServerRegistry.save(request.json) - else: - ServiceRegistry.save_service_info(**request.json) - return get_json_result(data=update_server) - - -@manager.route('/query', methods=['POST']) -@validate_request("service_name") -def get_service(): - service_info = ServiceRegistry.load_service(**request.json) - return get_json_result(data={"service_info": [service.to_json() for service in service_info]}) \ No newline at end of file diff --git a/python/fate_flow/apps/table_app.py b/python/fate_flow/apps/table_app.py deleted file mode 100644 index 54f2248fb..000000000 --- a/python/fate_flow/apps/table_app.py +++ /dev/null @@ -1,369 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_arch import storage -from fate_arch.metastore.db_utils import StorageConnector -from fate_arch.session import Session -from fate_arch.storage import StorageTableMeta, StorageTableOrigin -from fate_flow.entity import RunParameters -from fate_flow.manager.data_manager import DataTableTracker, TableStorage, SchemaMetaParam, AnonymousGenerator -from fate_flow.operation.job_saver import JobSaver -from fate_flow.operation.job_tracker import Tracker -from fate_flow.utils.data_utils import get_extend_id_name, address_filter -from fate_flow.worker.task_executor import TaskExecutor -from fate_flow.utils.api_utils import get_json_result, error_response, validate_request -from fate_flow.utils import job_utils, schedule_utils -from flask import request - - -@manager.route('/connector/create', methods=['POST']) -def create_storage_connector(): - request_data = request.json - address = StorageTableMeta.create_address(request_data.get("engine"), request_data.get("connector_info")) - connector = StorageConnector(connector_name=request_data.get("connector_name"), engine=request_data.get("engine"), - connector_info=address.connector) - connector.create_or_update() - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('/connector/query', methods=['POST']) -def query_storage_connector(): - request_data = request.json - connector = StorageConnector(connector_name=request_data.get("connector_name")) - return get_json_result(retcode=0, retmsg='success', data=connector.get_info()) - - -@manager.route('/add', methods=['post']) -@manager.route('/bind', methods=['post']) -@validate_request("engine", "address", "namespace", "name") -def table_bind(): - request_data = request.json - address_dict = request_data.get('address') - engine = request_data.get('engine') - name = request_data.get('name') - namespace = request_data.get('namespace') - extra_schema = request_data.get("schema", {}) - address = storage.StorageTableMeta.create_address(storage_engine=engine, address_dict=address_dict) - in_serialized = request_data.get("in_serialized", 1 if engine in {storage.StorageEngine.STANDALONE, storage.StorageEngine.EGGROLL, - storage.StorageEngine.MYSQL, storage.StorageEngine.PATH, - storage.StorageEngine.API} else 0) - destroy = (int(request_data.get("drop", 0)) == 1) - data_table_meta = storage.StorageTableMeta(name=name, namespace=namespace) - if data_table_meta: - if destroy: - data_table_meta.destroy_metas() - else: - return get_json_result(retcode=100, - retmsg='The data table already exists.' - 'If you still want to continue uploading, please add the parameter --drop') - id_column = request_data.get("id_column") or request_data.get("id_name") - feature_column = request_data.get("feature_column") or request_data.get("feature_name") - schema = get_bind_table_schema(id_column, feature_column) - schema.update(extra_schema) - if request_data.get("with_meta", False): - meta = SchemaMetaParam(**request_data.get("meta", {})) - if request_data.get("extend_sid", False): - meta.with_match_id = True - schema.update({"meta": meta.to_dict()}) - extra_schema["meta"] = meta.to_dict() - sess = Session() - storage_session = sess.storage(storage_engine=engine, options=request_data.get("options")) - table = storage_session.create_table(address=address, name=name, namespace=namespace, - partitions=request_data.get('partitions', None), - have_head=request_data.get("head"), schema=schema, - extend_sid=request_data.get("extend_sid", False), - id_delimiter=request_data.get("id_delimiter"), - in_serialized=in_serialized, - origin=request_data.get("origin", StorageTableOrigin.TABLE_BIND) - ) - response = get_json_result(data={"table_name": name, "namespace": namespace}) - if not table.check_address(): - response = get_json_result(retcode=100, retmsg=f'engine {engine} address {address_dict} check failed') - else: - if request_data.get("extend_sid"): - schema = update_bind_table_schema(id_column, feature_column, request_data.get("extend_sid"), request_data.get("id_delimiter")) - schema.update(extra_schema) - table.meta.update_metas(schema=schema) - DataTableTracker.create_table_tracker( - table_name=name, - table_namespace=namespace, - entity_info={"have_parent": False}, - ) - sess.destroy_all_sessions() - return response - - -@manager.route('/schema/update', methods=['post']) -@validate_request("schema", "namespace", "name") -def schema_update(): - request_data = request.json - data_table_meta = storage.StorageTableMeta(name=request_data.get("name"), namespace=request_data.get("namespace")) - schema = data_table_meta.get_schema() - if request_data.get("schema", {}).get("meta"): - if schema.get("meta"): - schema = AnonymousGenerator.recover_schema(schema) - schema["meta"].update(request_data.get("schema").get("meta")) - else: - return get_json_result(retcode=101, retmsg="no found meta") - request_data["schema"].pop("meta", {}) - schema.update(request_data.get("schema", {})) - data_table_meta.update_metas(schema=schema) - return get_json_result(data=schema) - - -@manager.route('/schema/anonymous/migrate', methods=['post']) -@validate_request("namespace", "name", "role", "party_id", "migrate_mapping") -def meta_update(): - request_data = request.json - data_table_meta = storage.StorageTableMeta(name=request_data.get("name"), namespace=request_data.get("namespace")) - schema = data_table_meta.get_schema() - update_schema = AnonymousGenerator.migrate_schema_anonymous( - anonymous_schema=schema, - role=request_data.get("role"), - party_id=request_data.get("party_id"), - migrate_mapping=request_data.get("migrate_mapping")) - if update_schema: - schema.update(update_schema) - data_table_meta.update_metas(schema=schema) - return get_json_result(data=schema) - else: - return get_json_result(retcode=101, retmsg="update failed") - - -@manager.route('/download', methods=['get']) -def table_download(): - request_data = request.json - from fate_flow.component_env_utils.env_utils import import_component_output_depend - import_component_output_depend() - data_table_meta = storage.StorageTableMeta(name=request_data.get("name"), namespace=request_data.get("namespace")) - if not data_table_meta: - return error_response(response_code=210, retmsg=f'no found table:{request_data.get("namespace")}, {request_data.get("name")}') - tar_file_name = 'table_{}_{}.tar.gz'.format(request_data.get("namespace"), request_data.get("name")) - return TableStorage.send_table( - output_tables_meta={"table": data_table_meta}, - tar_file_name=tar_file_name, - need_head=request_data.get("head", True) - ) - - -@manager.route('/preview', methods=['post']) -def table_data_preview(): - request_data = request.json - from fate_flow.component_env_utils.env_utils import import_component_output_depend - import_component_output_depend() - data_table_meta = storage.StorageTableMeta(name=request_data.get("name"), namespace=request_data.get("namespace")) - if not data_table_meta: - return error_response(response_code=210, retmsg=f'no found table:{request_data.get("namespace")}, {request_data.get("name")}') - - data = TableStorage.read_table_data(data_table_meta, limit=request_data.get("limit")) - return get_json_result(retcode=0, retmsg='success', data=data) - - -@manager.route('/delete', methods=['post']) -def table_delete(): - request_data = request.json - table_name = request_data.get('table_name') - namespace = request_data.get('namespace') - data = None - sess = Session() - table = sess.get_table(name=table_name, namespace=namespace, ignore_disable=True) - if table: - table.destroy() - data = {'table_name': table_name, 'namespace': namespace} - sess.destroy_all_sessions() - if data: - return get_json_result(data=data) - return get_json_result(retcode=101, retmsg='no find table') - - -@manager.route('/disable', methods=['post']) -@manager.route('/enable', methods=['post']) -def table_disable(): - request_data = request.json - adapter_request_data(request_data) - disable = True if request.url.endswith("disable") else False - tables_meta = storage.StorageTableMeta.query_table_meta(filter_fields=dict(**request_data)) - data = [] - if tables_meta: - for table_meta in tables_meta: - storage.StorageTableMeta(name=table_meta.f_name, - namespace=table_meta.f_namespace - ).update_metas(disable=disable) - data.append({'table_name': table_meta.f_name, 'namespace': table_meta.f_namespace}) - return get_json_result(data=data) - return get_json_result(retcode=101, retmsg='no find table') - - -@manager.route('/disable/delete', methods=['post']) -def table_delete_disable(): - request_data = request.json - adapter_request_data(request_data) - tables_meta = storage.StorageTableMeta.query_table_meta(filter_fields={"disable": True}) - data = [] - sess = Session() - for table_meta in tables_meta: - table = sess.get_table(name=table_meta.f_name, namespace=table_meta.f_namespace, ignore_disable=True) - if table: - table.destroy() - data.append({'table_name': table_meta.f_name, 'namespace': table_meta.f_namespace}) - sess.destroy_all_sessions() - if data: - return get_json_result(data=data) - return get_json_result(retcode=101, retmsg='no find table') - - -@manager.route('/list', methods=['post']) -@validate_request('job_id', 'role', 'party_id') -def get_job_table_list(): - jobs = JobSaver.query_job(**request.json) - if jobs: - job = jobs[0] - tables = get_job_all_table(job) - return get_json_result(data=tables) - else: - return get_json_result(retcode=101, retmsg='no find job') - - -@manager.route('/', methods=['post']) -def table_api(table_func): - config = request.json - if table_func == 'table_info': - table_key_count = 0 - table_partition = None - table_schema = None - extend_sid = False - table_name, namespace = config.get("name") or config.get("table_name"), config.get("namespace") - table_meta = storage.StorageTableMeta(name=table_name, namespace=namespace) - address = None - enable = True - origin = None - if table_meta: - table_key_count = table_meta.get_count() - table_partition = table_meta.get_partitions() - table_schema = table_meta.get_schema() - extend_sid = table_meta.get_extend_sid() - table_schema.update() - address = address_filter(table_meta.get_address()) - enable = not table_meta.get_disable() - origin = table_meta.get_origin() - exist = 1 - else: - exist = 0 - return get_json_result(data={"table_name": table_name, - "namespace": namespace, - "exist": exist, - "count": table_key_count, - "partition": table_partition, - "schema": table_schema, - "enable": enable, - "origin": origin, - "extend_sid": extend_sid, - "address": address}) - else: - return get_json_result() - - -@manager.route('/tracking/source', methods=['post']) -@validate_request("table_name", "namespace") -def table_tracking(): - request_info = request.json - data = DataTableTracker.get_parent_table(request_info.get("table_name"), request_info.get("namespace")) - return get_json_result(data=data) - - -@manager.route('/tracking/job', methods=['post']) -@validate_request("table_name", "namespace") -def table_tracking_job(): - request_info = request.json - data = DataTableTracker.track_job(request_info.get("table_name"), request_info.get("namespace"), display=True) - return get_json_result(data=data) - - -def get_job_all_table(job): - dsl_parser = schedule_utils.get_job_dsl_parser(dsl=job.f_dsl, - runtime_conf=job.f_runtime_conf, - train_runtime_conf=job.f_train_runtime_conf - ) - _, hierarchical_structure = dsl_parser.get_dsl_hierarchical_structure() - component_table = {} - try: - component_output_tables = Tracker.query_output_data_infos(job_id=job.f_job_id, role=job.f_role, - party_id=job.f_party_id) - except: - component_output_tables = [] - for component_name_list in hierarchical_structure: - for component_name in component_name_list: - component_table[component_name] = {} - component_input_table = get_component_input_table(dsl_parser, job, component_name) - component_table[component_name]['input'] = component_input_table - component_table[component_name]['output'] = {} - for output_table in component_output_tables: - if output_table.f_component_name == component_name: - component_table[component_name]['output'][output_table.f_data_name] = \ - {'name': output_table.f_table_name, 'namespace': output_table.f_table_namespace} - return component_table - - -def get_component_input_table(dsl_parser, job, component_name): - component = dsl_parser.get_component_info(component_name=component_name) - module_name = get_component_module(component_name, job.f_dsl) - if 'reader' == module_name.lower(): - return job.f_runtime_conf.get("component_parameters", {}).get("role", {}).get(job.f_role, {}).get(str(job.f_roles.get(job.f_role).index(int(job.f_party_id)))).get(component_name) - task_input_dsl = component.get_input() - job_args_on_party = TaskExecutor.get_job_args_on_party(dsl_parser=dsl_parser, - job_runtime_conf=job.f_runtime_conf, role=job.f_role, - party_id=job.f_party_id) - config = job_utils.get_job_parameters(job.f_job_id, job.f_role, job.f_party_id) - task_parameters = RunParameters(**config) - job_parameters = task_parameters - component_input_table = TaskExecutor.get_task_run_args(job_id=job.f_job_id, role=job.f_role, - party_id=job.f_party_id, - task_id=None, - task_version=None, - job_args=job_args_on_party, - job_parameters=job_parameters, - task_parameters=task_parameters, - input_dsl=task_input_dsl, - get_input_table=True - ) - return component_input_table - - -def get_component_module(component_name, job_dsl): - return job_dsl["components"][component_name]["module"].lower() - - -def adapter_request_data(request_data): - if request_data.get("table_name"): - request_data["name"] = request_data.get("table_name") - - -def get_bind_table_schema(id_column, feature_column): - schema = {} - if id_column and feature_column: - schema = {'header': feature_column, 'sid': id_column} - elif id_column: - schema = {'sid': id_column, 'header': ''} - return schema - - -def update_bind_table_schema(id_column, feature_column, extend_sid, id_delimiter): - schema = None - if id_column and feature_column: - schema = {'header': id_delimiter.join([id_column, feature_column]), 'sid': get_extend_id_name()} - elif id_column: - schema = {'header': id_column, 'sid': get_extend_id_name()} - schema.update({'extend_tag': True}) - return schema diff --git a/python/fate_flow/apps/template_app.py b/python/fate_flow/apps/template_app.py deleted file mode 100644 index 41204a756..000000000 --- a/python/fate_flow/apps/template_app.py +++ /dev/null @@ -1,55 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import io -import os -import tarfile - -from flask import send_file, request - -from fate_arch.common import file_utils -from fate_flow.settings import TEMPLATE_INFO_PATH -from fate_flow.utils.base_utils import get_fate_flow_directory - - -@manager.route('/download', methods=['post']) -def template_download(): - min_data = request.json.get("min_data", False) if request.json else False - memory_file = io.BytesIO() - dir_dict = {} - template_info = file_utils.load_yaml_conf(TEMPLATE_INFO_PATH) - data_dir = template_info.get("template_data", {}).get("base_dir") - min_data_file = template_info.get("template_data", {}).get("min_data", []) - for name, dir_name in template_info.get("template_path", {}).items(): - dir_dict[name] = os.path.join(get_fate_flow_directory(), dir_name) - delete_dir_list = [] - for name, dir_list in template_info.get("delete_path").items(): - for dir_name in dir_list: - delete_dir_list.append(os.path.join(dir_dict[name], dir_name)) - tar = tarfile.open(fileobj=memory_file, mode='w:gz') - for name, base_dir in dir_dict.items(): - for root, dir, files in os.walk(base_dir): - for file in files: - if min_data: - if data_dir in root and file not in min_data_file: - continue - if root in delete_dir_list: - continue - full_path = os.path.join(root, file) - rel_path = os.path.join(name, os.path.relpath(full_path, base_dir)) - tar.add(full_path, rel_path) - tar.close() - memory_file.seek(0) - return send_file(memory_file, attachment_filename=f'template.tar.gz', as_attachment=True) \ No newline at end of file diff --git a/python/fate_flow/apps/tracking_app.py b/python/fate_flow/apps/tracking_app.py deleted file mode 100644 index a3d4110d9..000000000 --- a/python/fate_flow/apps/tracking_app.py +++ /dev/null @@ -1,338 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import json -import os - -from flask import jsonify, request, send_file - -from fate_flow.component_env_utils import feature_utils -from fate_flow.component_env_utils.env_utils import import_component_output_depend -from fate_flow.db.db_models import DB, Job -from fate_flow.manager.data_manager import TableStorage, delete_metric_data, get_component_output_data_schema -from fate_flow.model.sync_model import SyncComponent -from fate_flow.operation.job_saver import JobSaver -from fate_flow.operation.job_tracker import Tracker -from fate_flow.scheduler.federated_scheduler import FederatedScheduler -from fate_flow.settings import TEMP_DIRECTORY, stat_logger, ENABLE_MODEL_STORE -from fate_flow.utils import job_utils, schedule_utils -from fate_flow.utils.api_utils import error_response, get_json_result, validate_request - - -@manager.route('/job/data_view', methods=['post']) -def job_view(): - request_data = request.json - check_request_parameters(request_data) - job_tracker = Tracker(job_id=request_data['job_id'], role=request_data['role'], party_id=request_data['party_id']) - job_view_data = job_tracker.get_job_view() - if job_view_data: - job_metric_list = job_tracker.get_metric_list(job_level=True) - job_view_data['model_summary'] = {} - for metric_namespace, namespace_metrics in job_metric_list.items(): - job_view_data['model_summary'][metric_namespace] = job_view_data['model_summary'].get(metric_namespace, {}) - for metric_name in namespace_metrics: - job_view_data['model_summary'][metric_namespace][metric_name] = job_view_data['model_summary'][ - metric_namespace].get(metric_name, {}) - for metric_data in job_tracker.get_job_metric_data(metric_namespace=metric_namespace, - metric_name=metric_name): - job_view_data['model_summary'][metric_namespace][metric_name][metric_data.key] = metric_data.value - return get_json_result(retcode=0, retmsg='success', data=job_view_data) - else: - return get_json_result(retcode=101, retmsg='error') - - -@manager.route('/component/metric/all', methods=['post']) -def component_metric_all(): - request_data = request.json - check_request_parameters(request_data) - tracker = Tracker(job_id=request_data['job_id'], component_name=request_data['component_name'], - role=request_data['role'], party_id=request_data['party_id']) - metrics = tracker.get_metric_list() - all_metric_data = {} - if metrics: - for metric_namespace, metric_names in metrics.items(): - all_metric_data[metric_namespace] = all_metric_data.get(metric_namespace, {}) - for metric_name in metric_names: - all_metric_data[metric_namespace][metric_name] = all_metric_data[metric_namespace].get(metric_name, {}) - metric_data, metric_meta = get_metric_all_data(tracker=tracker, metric_namespace=metric_namespace, - metric_name=metric_name) - all_metric_data[metric_namespace][metric_name]['data'] = metric_data - all_metric_data[metric_namespace][metric_name]['meta'] = metric_meta - return get_json_result(retcode=0, retmsg='success', data=all_metric_data) - else: - return get_json_result(retcode=0, retmsg='no data', data={}) - - -@manager.route('/component/metrics', methods=['post']) -def component_metrics(): - request_data = request.json - check_request_parameters(request_data) - tracker = Tracker(job_id=request_data['job_id'], component_name=request_data['component_name'], - role=request_data['role'], party_id=request_data['party_id']) - metrics = tracker.get_metric_list() - if metrics: - return get_json_result(retcode=0, retmsg='success', data=metrics) - else: - return get_json_result(retcode=0, retmsg='no data', data={}) - - -@manager.route('/component/metric_data', methods=['post']) -def component_metric_data(): - request_data = request.json - check_request_parameters(request_data) - tracker = Tracker(job_id=request_data['job_id'], component_name=request_data['component_name'], - role=request_data['role'], party_id=request_data['party_id']) - metric_data, metric_meta = get_metric_all_data(tracker=tracker, metric_namespace=request_data['metric_namespace'], - metric_name=request_data['metric_name']) - if metric_data or metric_meta: - return get_json_result(retcode=0, retmsg='success', data=metric_data, - meta=metric_meta) - else: - return get_json_result(retcode=0, retmsg='no data', data=[], meta={}) - - -def get_metric_all_data(tracker, metric_namespace, metric_name): - metric_data = tracker.get_metric_data(metric_namespace=metric_namespace, - metric_name=metric_name) - metric_meta = tracker.get_metric_meta(metric_namespace=metric_namespace, - metric_name=metric_name) - if metric_data or metric_meta: - metric_data_list = [(metric.key, metric.value) for metric in metric_data] - metric_data_list.sort(key=lambda x: x[0]) - return metric_data_list, metric_meta.to_dict() if metric_meta else {} - else: - return [], {} - - -@manager.route('/component/metric/delete', methods=['post']) -def component_metric_delete(): - sql = delete_metric_data(request.json) - return get_json_result(retcode=0, retmsg='success', data=sql) - - -@manager.route('/component/parameters', methods=['post']) -def component_parameters(): - request_data = request.json - check_request_parameters(request_data) - tasks = JobSaver.query_task(only_latest=True, **request_data) - if not tasks: - return get_json_result(retcode=101, retmsg='can not found this task') - parameters = tasks[0].f_component_parameters - output_parameters = {} - output_parameters['module'] = parameters.get('module', '') - for p_k, p_v in parameters.items(): - if p_k.endswith('Param'): - output_parameters[p_k] = p_v - return get_json_result(retcode=0, retmsg='success', data=output_parameters) - - -@manager.route('/component/output/model', methods=['post']) -def component_output_model(): - request_data = request.json - check_request_parameters(request_data) - - job_configuration = job_utils.get_job_configuration(request_data['job_id'], request_data['role'], request_data['party_id']) - - model_id = job_configuration.runtime_conf_on_party['job_parameters']['model_id'] - model_version = request_data['job_id'] - - tracker = Tracker( - job_id=request_data['job_id'], - role=request_data['role'], party_id=request_data['party_id'], - model_id=model_id, model_version=model_version, - component_name=request_data['component_name'], - ) - - define_meta = tracker.pipelined_model.pipelined_component.get_define_meta() - if not define_meta or request_data['component_name'] not in define_meta['component_define']: - return get_json_result(retcode=0, retmsg='no define_meta', data={}) - - component_define = define_meta['component_define'][request_data['component_name']] - # There is only one model output at the current dsl version. - model_alias = next(iter(define_meta['model_proto'][request_data['component_name']].keys())) - - if ENABLE_MODEL_STORE: - sync_component = SyncComponent( - role=request_data['role'], - party_id=request_data['party_id'], - model_id=model_id, - model_version=model_version, - component_name=request_data['component_name'], - ) - if not sync_component.local_exists() and sync_component.remote_exists(): - sync_component.download() - - output_model = tracker.pipelined_model.read_component_model( - component_name=request_data['component_name'], - model_alias=model_alias, - output_json=True, - ) - - output_model_json = {} - component_model_meta = {} - - for buffer_name, buffer_object_json_format in output_model.items(): - if buffer_name.endswith('Param'): - output_model_json = buffer_object_json_format - elif buffer_name.endswith('Meta'): - component_model_meta = { - 'meta_data': buffer_object_json_format, - } - - if not output_model_json: - return get_json_result(retcode=0, retmsg='no data', data={}) - - component_model_meta.update(component_define) - - return get_json_result(retcode=0, retmsg='success', data=output_model_json, meta=component_model_meta) - - -@manager.route('/component/output/data', methods=['post']) -def component_output_data(): - request_data = request.json - tasks = JobSaver.query_task(only_latest=True, job_id=request_data['job_id'], - component_name=request_data['component_name'], - role=request_data['role'], party_id=request_data['party_id']) - if not tasks: - raise ValueError(f'no found task, please check if the parameters are correct:{request_data}') - import_component_output_depend(tasks[0].f_provider_info) - output_tables_meta = get_component_output_tables_meta(task_data=request_data) - if not output_tables_meta: - return get_json_result(retcode=0, retmsg='no data', data=[]) - output_data_list = [] - headers = [] - totals = [] - data_names = [] - for output_name, output_table_meta in output_tables_meta.items(): - output_data = [] - is_str = False - all_extend_header = {} - if output_table_meta: - for k, v in output_table_meta.get_part_of_data(): - data_line, is_str, all_extend_header = feature_utils.get_component_output_data_line(src_key=k, src_value=v, schema=output_table_meta.get_schema(), all_extend_header=all_extend_header) - output_data.append(data_line) - total = output_table_meta.get_count() - output_data_list.append(output_data) - data_names.append(output_name) - totals.append(total) - if output_data: - extend_header = feature_utils.generate_header(all_extend_header, schema=output_table_meta.get_schema()) - if output_table_meta.schema.get("is_display", True): - header = get_component_output_data_schema(output_table_meta=output_table_meta, is_str=is_str, - extend_header=extend_header) - else: - header = [] - headers.append(header) - else: - headers.append(None) - if len(output_data_list) == 1 and not output_data_list[0]: - return get_json_result(retcode=0, retmsg='no data', data=[]) - return get_json_result(retcode=0, retmsg='success', data=output_data_list, - meta={'header': headers, 'total': totals, 'names': data_names}) - - -@manager.route('/component/output/data/download', methods=['get']) -def component_output_data_download(): - request_data = request.json - tasks = JobSaver.query_task(only_latest=True, job_id=request_data['job_id'], - component_name=request_data['component_name'], - role=request_data['role'], party_id=request_data['party_id']) - if not tasks: - raise ValueError(f'no found task, please check if the parameters are correct:{request_data}') - import_component_output_depend(tasks[0].f_provider_info) - try: - output_tables_meta = get_component_output_tables_meta(task_data=request_data) - except Exception as e: - stat_logger.exception(e) - return error_response(210, str(e)) - limit = request_data.get('limit', -1) - if not output_tables_meta: - return error_response(response_code=210, retmsg='no data') - if limit == 0: - return error_response(response_code=210, retmsg='limit is 0') - tar_file_name = 'job_{}_{}_{}_{}_output_data.tar.gz'.format(request_data['job_id'], - request_data['component_name'], - request_data['role'], request_data['party_id']) - return TableStorage.send_table(output_tables_meta, tar_file_name, limit=limit, need_head=request_data.get("head", True)) - - -@manager.route('/component/output/data/table', methods=['post']) -@validate_request('job_id', 'role', 'party_id', 'component_name') -def component_output_data_table(): - request_data = request.json - jobs = JobSaver.query_job(job_id=request_data.get('job_id')) - if jobs: - job = jobs[0] - return jsonify(FederatedScheduler.tracker_command(job, request_data, 'output/table')) - else: - return get_json_result(retcode=100, retmsg='No found job') - - -@manager.route('/component/summary/download', methods=['POST']) -@validate_request("job_id", "component_name", "role", "party_id") -def get_component_summary(): - request_data = request.json - try: - tracker = Tracker(job_id=request_data["job_id"], component_name=request_data["component_name"], - role=request_data["role"], party_id=request_data["party_id"], - task_id=request_data.get("task_id", None), task_version=request_data.get("task_version", None)) - summary = tracker.read_summary_from_db() - if summary: - if request_data.get("filename"): - temp_filepath = os.path.join(TEMP_DIRECTORY, request_data.get("filename")) - with open(temp_filepath, "w") as fout: - fout.write(json.dumps(summary, indent=4)) - return send_file(open(temp_filepath, "rb"), as_attachment=True, - attachment_filename=request_data.get("filename")) - else: - return get_json_result(data=summary) - return error_response(210, "No component summary found, please check if arguments are specified correctly.") - except Exception as e: - stat_logger.exception(e) - return error_response(210, str(e)) - - -@manager.route('/component/list', methods=['POST']) -def component_list(): - request_data = request.json - parser, _, _ = schedule_utils.get_job_dsl_parser_by_job_id(job_id=request_data.get('job_id')) - if parser: - return get_json_result(data={'components': list(parser.get_dsl().get('components').keys())}) - else: - return get_json_result(retcode=100, retmsg='No job matched, please make sure the job id is valid.') - - -def get_component_output_tables_meta(task_data): - check_request_parameters(task_data) - tracker = Tracker(job_id=task_data['job_id'], component_name=task_data['component_name'], - role=task_data['role'], party_id=task_data['party_id']) - output_data_table_infos = tracker.get_output_data_info() - output_tables_meta = tracker.get_output_data_table(output_data_infos=output_data_table_infos) - return output_tables_meta - - -@DB.connection_context() -def check_request_parameters(request_data): - if 'role' not in request_data or 'party_id' not in request_data: - jobs = Job.select(Job.f_runtime_conf_on_party).where(Job.f_job_id == request_data.get('job_id', ''), - Job.f_is_initiator == True) - if jobs: - job = jobs[0] - job_runtime_conf = job.f_runtime_conf_on_party - job_initiator = job_runtime_conf.get('initiator', {}) - role = job_initiator.get('role', '') - party_id = job_initiator.get('party_id', 0) - request_data['role'] = role if 'role' not in request_data else request_data['role'] - request_data['party_id'] = party_id if 'party_id' not in request_data else request_data['party_id'] diff --git a/python/fate_flow/pipelined_model/homo_model_deployer/__init__.py b/python/fate_flow/apps/worker/__init__.py similarity index 90% rename from python/fate_flow/pipelined_model/homo_model_deployer/__init__.py rename to python/fate_flow/apps/worker/__init__.py index 039425cb6..ae946a49c 100644 --- a/python/fate_flow/pipelined_model/homo_model_deployer/__init__.py +++ b/python/fate_flow/apps/worker/__init__.py @@ -1,5 +1,5 @@ # -# Copyright 2021 The FATE Authors. All Rights Reserved. +# Copyright 2019 The FATE Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,4 +12,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# diff --git a/python/fate_flow/apps/worker/worker_app.py b/python/fate_flow/apps/worker/worker_app.py new file mode 100644 index 000000000..295d1ce40 --- /dev/null +++ b/python/fate_flow/apps/worker/worker_app.py @@ -0,0 +1,193 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask import request +from webargs import fields + +from fate_flow.controller.task import TaskController +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import NoFoundTask +from fate_flow.manager.outputs.data import DataManager, OutputDataTracking +from fate_flow.manager.outputs.model import PipelinedModel +from fate_flow.manager.outputs.metric import OutputMetric +from fate_flow.manager.operation.job_saver import JobSaver +from fate_flow.utils.api_utils import API + +page_name = 'worker' + + +@manager.route('/task/status', methods=['POST']) +@API.Input.json(execution_id=fields.String(required=True)) +@API.Input.json(status=fields.String(required=True)) +@API.Input.json(error=fields.String(required=False)) +@API.Output.runtime_exception(code=ReturnCode.API.COMPONENT_OUTPUT_EXCEPTION) +def report_task_status(status, execution_id, error=None): + task = JobSaver.query_task_by_execution_id(execution_id=execution_id) + task_info = { + "party_status": status, + "job_id": task.f_job_id, + "role": task.f_role, + "party_id": task.f_party_id, + "task_id": task.f_task_id, + "task_version": task.f_task_version + } + TaskController.update_task_status(task_info=task_info) + if error: + task_info.update({"error_report": error}) + TaskController.update_task(task_info) + return API.Output.json() + + +@manager.route('/model/save', methods=['POST']) +@API.Input.form(model_id=fields.String(required=True)) +@API.Input.form(model_version=fields.String(required=True)) +@API.Input.form(execution_id=fields.String(required=True)) +@API.Input.form(output_key=fields.String(required=True)) +@API.Input.form(type_name=fields.String(required=True)) +@API.Output.runtime_exception(code=ReturnCode.API.COMPONENT_OUTPUT_EXCEPTION) +def upload_model(model_id, model_version, execution_id, output_key, type_name): + task = JobSaver.query_task_by_execution_id(execution_id=execution_id) + file = request.files['file'] + PipelinedModel.upload_model( + model_file=file, + job_id=task.f_job_id, + task_name=task.f_task_name, + role=task.f_role, + party_id=task.f_party_id, + output_key=output_key, + model_id=model_id, + model_version=model_version, + type_name=type_name + ) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success") + + +@manager.route('/model/download', methods=['GET']) +@API.Input.params(model_id=fields.String(required=True)) +@API.Input.params(model_version=fields.String(required=True)) +@API.Input.params(role=fields.String(required=True)) +@API.Input.params(party_id=fields.String(required=True)) +@API.Input.params(task_name=fields.String(required=True)) +@API.Input.params(output_key=fields.String(required=True)) +@API.Output.runtime_exception(code=ReturnCode.API.COMPONENT_OUTPUT_EXCEPTION) +def download_model(model_id, model_version, role, party_id, task_name, output_key): + return PipelinedModel.download_model( + model_id=model_id, + model_version=model_version, + role=role, + party_id=party_id, + task_name=task_name, + output_key=output_key + ) + + +@manager.route('/data/tracking/query', methods=['GET']) +@API.Input.params(job_id=fields.String(required=False)) +@API.Input.params(role=fields.String(required=False)) +@API.Input.params(party_id=fields.String(required=False)) +@API.Input.params(task_name=fields.String(required=False)) +@API.Input.params(output_key=fields.String(required=False)) +@API.Input.params(namespace=fields.String(required=False)) +@API.Input.params(name=fields.String(required=False)) +@API.Output.runtime_exception(code=ReturnCode.API.COMPONENT_OUTPUT_EXCEPTION) +def query_data_tracking(job_id=None, role=None, party_id=None, task_name=None, output_key=None, namespace=None, name=None): + tracking_list = [] + if not namespace and not name: + data_info = { + "job_id": job_id, + "role": role, + "party_id": party_id, + "task_name": task_name, + "output_key": output_key + } + tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + if not tasks: + return API.Output.fate_flow_exception(e=NoFoundTask(job_id=job_id, role=role, party_id=party_id, + task_name=task_name)) + data_info.update({ + "task_id": tasks[0].f_task_id, + "task_version": tasks[0].f_task_version + }) + + data_list = OutputDataTracking.query(**data_info) + if not data_list: + return API.Output.json(code=ReturnCode.Task.NO_FOUND_MODEL_OUTPUT, message="failed") + for data in data_list: + info, _ = DataManager.get_data_info(data.f_namespace, data.f_name) + tracking_list.append(info) + else: + info, _ = DataManager.get_data_info(namespace, name) + tracking_list.append(info) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success", data=tracking_list) + + +@manager.route('/data/tracking/save', methods=['POST']) +@API.Input.json(execution_id=fields.String(required=True)) +@API.Input.json(meta_data=fields.Dict(required=True)) +@API.Input.json(uri=fields.String(required=True)) +@API.Input.json(output_key=fields.String(required=True)) +@API.Input.json(namespace=fields.String(required=True)) +@API.Input.json(name=fields.String(required=True)) +@API.Input.json(overview=fields.Dict(required=True)) +@API.Input.json(partitions=fields.Int(required=False)) +@API.Input.json(source=fields.Dict(required=True)) +@API.Input.json(data_type=fields.String(required=True)) +@API.Input.json(index=fields.Int(required=True)) +@API.Output.runtime_exception(code=ReturnCode.API.COMPONENT_OUTPUT_EXCEPTION) +def save_data_tracking(execution_id, meta_data, uri, output_key, namespace, name, overview, source, data_type, index, + partitions=None): + task = JobSaver.query_task_by_execution_id(execution_id=execution_id) + data_info = { + "uri": uri, + "output_key": output_key, + "job_id": task.f_job_id, + "role": task.f_role, + "party_id": task.f_party_id, + "task_id": task.f_task_id, + "task_version": task.f_task_version, + "task_name": task.f_task_name, + "namespace": namespace, + "name": name, + "index": index + } + OutputDataTracking.create(data_info) + if uri: + DataManager.create_data_table( + namespace=namespace, name=name, uri=uri, partitions=partitions, + data_meta=meta_data, source=source, data_type=data_type, + count=overview.get("count", None), part_of_data=overview.get("samples", []) + ) + return API.Output.json(code=ReturnCode.Base.SUCCESS, message="success") + + +@manager.route('/metric/save', methods=["POST"]) +@API.Input.json(execution_id=fields.String(required=True)) +@API.Input.json(data=fields.List(fields.Dict())) +@API.Output.runtime_exception(code=ReturnCode.API.COMPONENT_OUTPUT_EXCEPTION) +def save_metric(execution_id, data): + task = JobSaver.query_task_by_execution_id(execution_id=execution_id) + OutputMetric(job_id=task.f_job_id, role=task.f_role, party_id=task.f_party_id, task_name=task.f_task_name, + task_id=task.f_task_id, task_version=task.f_task_version).save_output_metrics(data) + return API.Output.json() + + +@manager.route('/metric/save/', methods=["POST"]) +@API.Input.json(data=fields.List(fields.Dict())) +@API.Output.runtime_exception(code=ReturnCode.API.COMPONENT_OUTPUT_EXCEPTION) +def save_metrics(execution_id, data): + task = JobSaver.query_task_by_execution_id(execution_id=execution_id) + OutputMetric(job_id=task.f_job_id, role=task.f_role, party_id=task.f_party_id, task_name=task.f_task_name, + task_id=task.f_task_id, task_version=task.f_task_version).save_output_metrics(data) + return API.Output.json() diff --git a/python/fate_flow/commands/__init__.py b/python/fate_flow/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/commands/server_cli.py b/python/fate_flow/commands/server_cli.py new file mode 100644 index 000000000..83bc3e24e --- /dev/null +++ b/python/fate_flow/commands/server_cli.py @@ -0,0 +1,200 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import subprocess +import platform +import click +from ruamel import yaml + +import fate_flow +from fate_flow.commands.service import manage_fate_service + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +HOME = os.path.dirname(fate_flow.__file__) +SERVER_CONF_PATH = os.path.join(HOME, "conf", "service_conf.yaml") +SETTING_PATH = os.path.join(HOME, "settings.py") +SERVICE_SH = os.path.join(HOME, "commands", "service.sh") + + +@click.group(short_help='Fate Flow', context_settings=CONTEXT_SETTINGS) +@click.pass_context +def flow_server_cli(ctx): + ''' + Fate Flow server cli + ''' + ctx.ensure_object(dict) + if ctx.invoked_subcommand == 'init': + return + pass + + +@flow_server_cli.command('init', short_help='Flow Server init command') +@click.option('--ip', type=click.STRING, help='Fate flow server ip address.') +@click.option('--port', type=click.INT, help='Fate flow server http port.') +@click.option('--home', type=click.STRING, help="Service's home directory, used to store essential information " + "such as data, logs, and more.") +def initialization(**kwargs): + """ + \b + - DESCRIPTION: + Flow Init Command. provide ip and port of a valid fate flow server. + + \b + - USAGE: + fate_flow init --ip 127.0.0.1 --port 9380 --home /data/projects/fate_flow + + """ + init_server(kwargs.get("ip"), kwargs.get("port"), kwargs.get("home")) + + +@flow_server_cli.command('start', short_help='Start run flow server') +def start(**kwargs): + """ + \b + - DESCRIPTION: + Start FATE Flow Server Command. + + \b + - USAGE: + fate_flow start + + """ + if platform.system().lower() == 'windows': + manage_fate_service(HOME, "start") + else: + run_command("start") + + +@flow_server_cli.command('status', short_help='Query fate flow server status') +def status(**kwargs): + """ + \b + - DESCRIPTION: + Query fate flow server status command + + \b + - USAGE: + fate_flow status + + """ + if platform.system().lower() == 'windows': + manage_fate_service(HOME, "status") + else: + run_command("status") + + +@flow_server_cli.command('stop', short_help='Stop run flow server') +def stop(**kwargs): + """ + \b + - DESCRIPTION: + Stop FATE Flow Server Command. + + \b + - USAGE: + fate_flow stop + + """ + if platform.system().lower() == 'windows': + manage_fate_service(HOME, "stop") + else: + run_command("stop") + + +@flow_server_cli.command('restart', short_help='Restart fate flow server') +def restart(**kwargs): + """ + \b + - DESCRIPTION: + ReStart FATE Flow Server Command. + + \b + - USAGE: + fate_flow restart + + """ + if platform.system().lower() == 'windows': + manage_fate_service(HOME, "restart") + else: + run_command("restart") + + +@flow_server_cli.command('version', short_help='Flow Server Version Command') +def get_version(): + import fate_flow + print(fate_flow.__version__) + + +def replace_settings(home_path): + import re + with open(SETTING_PATH, "r") as file: + content = file.read() + content = re.sub(r"DATA_DIR.*", f"DATA_DIR = \"{home_path}/data\"", content) + content = re.sub(r"MODEL_DIR.*", f"MODEL_DIR = \"{home_path}/model\"", content) + content = re.sub(r"JOB_DIR.*", f"JOB_DIR = \"{home_path}/jobs\"", content) + content = re.sub(r"LOG_DIR.*", f"LOG_DIR = \"{home_path}/logs\"", content) + content = re.sub(r"SQLITE_FILE_NAME.*", f"SQLITE_FILE_NAME = \"{home_path}/fate_flow_sqlite.db\"", content) + with open(SETTING_PATH, "w") as file: + file.write(content) + + with open(SERVICE_SH, "r") as file: + content = file.read() + content = re.sub(r"LOG_DIR.*=.*", f"LOG_DIR=\"{home_path}/logs\"", content) + with open(SERVICE_SH, "w") as file: + file.write(content) + + +def init_server(ip, port, home): + with open(SERVER_CONF_PATH, "r") as file: + config = yaml.safe_load(file) + if ip: + print(f"ip: {ip}") + config["fateflow"]["host"] = ip + if port: + print(f"port: {port}") + config["fateflow"]["http_port"] = port + if home: + if not os.path.isabs(home): + raise RuntimeError(f"Please use an absolute path: {home}") + os.makedirs(home, exist_ok=True) + print(f"home: {home}") + replace_settings(home) + + if ip or port: + with open(SERVER_CONF_PATH, "w") as file: + yaml.dump(config, file) + + print("Init server completed!") + + +def run_command(command): + try: + command = f"bash {SERVICE_SH} {HOME} {command}" + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, text=True) + if result.returncode == 0: + print(result.stdout) + return result.stdout + else: + print(result.stdout) + print(f"Error: {result.stderr}") + return None + except subprocess.CalledProcessError as e: + print(f"Error: {e}") + return command + + +if __name__ == '__main__': + flow_server_cli() diff --git a/python/fate_flow/commands/service.py b/python/fate_flow/commands/service.py new file mode 100644 index 000000000..6ea6cf3db --- /dev/null +++ b/python/fate_flow/commands/service.py @@ -0,0 +1,170 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import argparse +import os +import subprocess +import time +from ruamel import yaml + + +def load_yaml_conf(conf_path): + with open(conf_path) as f: + return yaml.safe_load(f) + + +def make_logs_dir(log_dir): + if not os.path.exists(log_dir): + os.makedirs(log_dir) + + +def manage_fate_service(project_base, action): + parser = argparse.ArgumentParser(description='FATE Service Manager') + parser.add_argument('project_base', type=str, help='path to the FATE project directory') + parser.add_argument('action', choices=['start', 'stop', 'status', 'restart'], help='action to perform') + + args = parser.parse_args([project_base, action]) + print(f'project_base:{args.project_base},action:{args.action}') + http_port, grpc_port = get_ports(args.project_base) + if args.action == 'start': + start_service(args.project_base) + get_service_status(http_port, grpc_port) + elif args.action == 'stop': + stop_service(http_port, grpc_port) + elif args.action == 'status': + get_service_status(http_port, grpc_port) + elif args.action == 'restart': + stop_service(http_port, grpc_port) + time.sleep(2) + start_service(args.project_base) + get_service_status(http_port, grpc_port) + + +def get_ports(project_base): + service_conf_path = os.path.join(project_base, 'conf/service_conf.yaml') + if not os.path.isfile(service_conf_path): + print(f'service conf not found: {service_conf_path}') + exit(1) + + config = load_yaml_conf(service_conf_path) + http_port = config.get('fateflow').get('http_port') + grpc_port = config.get('fateflow').get('grpc_port') + print(f'fate flow http port: {http_port}, grpc port: {grpc_port}\n') + return http_port, grpc_port + + +def get_pid(http_port, grpc_port): + netstat_command = ["netstat", "-ano"] + output = subprocess.run(netstat_command, capture_output=True, text=True).stdout + + pid = None + lines = output.split('\n') + for line in lines: + parts = line.split() + if len(parts) >= 5: + protocol = parts[0] + local_address = parts[1] + state = parts[3] + if state == 'LISTENING' and ':' in local_address: + port = local_address.split(':')[-1] + _pid = parts[-1] + if port == str(http_port) or port == str(grpc_port): + pid = _pid + break + return pid + + +def get_service_status(http_port, grpc_port): + pid = get_pid(http_port, grpc_port) + if pid: + task_list = subprocess.getoutput(f"tasklist /FI \"PID eq {pid}\"") + print(f"status: {task_list}") + + print(f'LISTENING on port {http_port}:') + print(subprocess.getoutput(f'netstat -ano | findstr :{http_port}')) + + print(f'LISTENING on port {grpc_port}:') + print(subprocess.getoutput(f'netstat -ano | findstr :{grpc_port}')) + else: + print('service not running') + + +def start_service(project_base): + http_port = None + grpc_port = None + + service_conf_path = os.path.join(project_base, 'conf/service_conf.yaml') + if os.path.isfile(service_conf_path): + config = load_yaml_conf(service_conf_path) + http_port = config.get('fateflow').get('http_port') + grpc_port = config.get('fateflow').get('grpc_port') + + if not http_port or not grpc_port: + print(f'service conf not found or missing port information: {service_conf_path}') + exit(1) + + pid = get_pid(http_port, grpc_port) + if pid: + print(f'service already started. pid: {pid}') + return + + log_dir = os.path.join(project_base, 'logs') + make_logs_dir(log_dir) + + command = ['python', os.path.join(project_base, 'fate_flow_server.py')] + # print(f'command:{command}') + stdout = open(os.path.join(log_dir, 'console.log'), 'a') + stderr = open(os.path.join(log_dir, 'error.log'), 'a') + + subprocess.Popen(command, stdout=stdout, stderr=stderr) + + for _ in range(100): + time.sleep(0.1) + pid = get_pid(http_port, grpc_port) + if pid: + print(f'service started successfully. pid: {pid}') + return + + pid = get_pid(http_port, grpc_port) + if pid: + print(f'service started successfully. pid: {pid}') + else: + print( + f'service start failed, please check {os.path.join(log_dir, "error.log")} and {os.path.join(log_dir, "console.log")}') + + +def stop_service(http_port, grpc_port): + pid = get_pid(http_port, grpc_port) + if not pid: + print('service not running') + return + task_list = subprocess.getoutput(f"tasklist /FI \"PID eq {pid}\"") + print(f'killing: {task_list}') + + try: + subprocess.run(['taskkill', '/F', '/PID', str(pid)]) + time.sleep(1) + except subprocess.CalledProcessError: + print('failed to kill the process') + return + + if get_pid(http_port, grpc_port): + print('failed to stop the service') + else: + print('service stopped successfully') + + + + diff --git a/python/fate_flow/commands/service.sh b/python/fate_flow/commands/service.sh new file mode 100644 index 000000000..ebcd442a4 --- /dev/null +++ b/python/fate_flow/commands/service.sh @@ -0,0 +1,404 @@ +#!/bin/bash +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ----------------------------------------------------------------------------- +# Service Control Script for a FATE Flow Server Application +# ----------------------------------------------------------------------------- +# +# This script is used to manage the lifecycle (start, stop, restart, and check status) of a server application. +# The server application listens on two ports: an HTTP port and a gRPC port. +# The settings for these ports, as well as other configurations, are read from a YAML file. +# +# Dependencies: +# - lsof: To check which processes are listening on which ports. +# - sed, awk: For text processing, mainly for parsing the YAML configuration. +# +# Usage: +# ./service.sh {start|stop|status|restart [sleep_time]} +# sleep_time: Optional. Number of seconds to wait between stop and start during restart. Default is 10 seconds. +# +# Assumptions: +# - The script assumes the presence of a configuration file named 'service_conf.yaml' in a relative directory. +# - The configuration file is structured in a specific way that the parsing logic expects. +# +# ----------------------------------------------------------------------------- +FATE_FLOW_BASE=$1 +LOG_DIR=$FATE_FLOW_BASE/logs + + +# --------------- Color Definitions --------------- +esc_c="\e[0m" +error_c="\e[31m" +ok_c="\e[32m" +highlight_c="\e[43m" + +# --------------- Logging Functions --------------- +print_info() { + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + local overwrite=$2 + + # Check if we need to overwrite the current line + if [ "$overwrite" == "overwrite" ]; then + echo -ne "\r${ok_c}[${timestamp}][MS]${esc_c} $1" + else + echo -e "${ok_c}[${timestamp}][MS]${esc_c} $1" + fi +} +print_ok() { + local overwrite=$2 + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + if [ "$overwrite" == "overwrite" ]; then + echo -ne "\r${ok_c}[${timestamp}][OK]${esc_c} $1" + else + echo -e "${ok_c}[${timestamp}][OK]${esc_c} $1" + fi +} + +print_error() { + local overwrite=$3 + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + if [ "$overwrite" == "overwrite" ]; then + echo -ne "\r${error_c}[${timestamp}][ER]${esc_c} $1: $2" + else + echo -e "${error_c}[${timestamp}][ER]${esc_c} $1: $2" + fi +} + +# --------------- Util Functions --------------- +# Check if the dependencies are installed on the system +check_dependencies() { + local missing_deps=0 + + for dep in lsof sed awk; do + if ! command -v "$dep" &>/dev/null; then + print_error "Missing dependency" "$dep" + missing_deps=1 + fi + done + + if [ "$missing_deps" -eq 1 ]; then + print_error "Please install the missing dependencies and try again." + exit 1 + fi +} + +# Get the PID of the process using a specific port +get_pid() { + local port=$1 + lsof -i:${port} | grep 'LISTEN' | awk 'NR==1 {print $2}' +} + +# Extract the specified port from a specified section of the YAML file +get_port_from_yaml() { + local yaml_file=$1 + local section=$2 + local port_key=$3 + local s='[[:space:]]*' + local w='[a-zA-Z0-9_]*' + local fs=$(echo @ | tr @ '\034') # Set fs back to the ASCII "file separator" + + sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ + -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $yaml_file | + awk -F$fs -v section="$section" -v port_key="$port_key" ' + { + indent = length($1)/2; + vname[indent] = $2; + for (i in vname) {if (i > indent) {delete vname[i]}} + if (length($3) > 0) { + vn=""; + for (i=0; i/dev/null; then + print_error "Process with PID ${pid} is not running." "" "overwrite" + echo + return 1 + fi + + if lsof -i :${http_port} | grep -q LISTEN && lsof -i :${grpc_port} | grep -q LISTEN; then + print_ok "Service started successfully!" "overwrite" + echo + return 0 + fi + + # Update spinning wheel + case $spin_state in + 0) spinner_char="/" ;; + 1) spinner_char="-" ;; + 2) spinner_char="\\" ;; + 3) spinner_char="|" ;; + esac + print_info "$spinner_char" "overwrite" + spin_state=$(((spin_state + 1) % 4)) + sleep $((interval_ms / 1000)).$((interval_ms % 1000)) + elapsed_ms=$((elapsed_ms + interval_ms)) + done + print_error "Service did not start up within the expected time." "" "overwrite" + echo + return 1 +} +# Draw a progress bar for visual feedback +draw_progress_bar() { + local completed=$1 + local total=$2 + local msg="$3" + local progress_bar="[" + + # Print completed part + for ((i = 0; i < completed; i++)); do + progress_bar+=" " + done + + # Print pending part + for ((i = completed; i < total; i++)); do + progress_bar+="-" + done + progress_bar+="]${msg}" + print_info "$progress_bar" "overwrite" +} + +# Checks if a port is active and returns the PID of the process using it. +# Parameters: +# $1 - The port number to check. +# Returns: +# PID of the process using the port, or an empty string if the port is not active. +check_port_active() { + local port=$1 + lsof -i:${port} | grep 'LISTEN' | awk 'NR==1 {print $2}' +} + +# Start service +start() { + print_info "--------------------------------starting--------------------------------" + print_info "Verifying if HTTP port ${highlight_c}${http_port}${esc_c} is not active..." + pid1=$(check_port_active $http_port) + if [ -n "${pid1}" ]; then + print_error "HTTP port ${highlight_c}${http_port}${esc_c} is already active. Process ID (PID): ${highlight_c}${pid1}${esc_c}" + exit 1 + else + print_ok "HTTP port ${highlight_c}${http_port}${esc_c} not active" + fi + + print_info "Verifying if gRPC port ${highlight_c}${grpc_port}${esc_c} is not active..." + pid2=$(check_port_active $grpc_port) + if [ -n "${pid2}" ]; then + print_error "gRPC port ${highlight_c}${grpc_port}${esc_c} is already active. Process ID (PID): ${highlight_c}${pid2}${esc_c}" + exit 1 + else + print_ok "gRPC port ${highlight_c}${grpc_port}${esc_c} not active" + fi + + print_info "Starting services..." + local startup_error_tmp=$(mktemp) + # nohup python "${FATE_FLOW_BASE}/fate_flow_server.py" >>"${LOG_STDOUT}" 2> >(tee -a "${LOG_STDERR}" >>"${startup_error_tmp}") & + nohup python $FATE_FLOW_BASE/fate_flow_server.py >> "${LOG_STDOUT}" 2>>"${LOG_STDERR}" & + pid=$! + print_info "Process ID (PID): ${highlight_c}${pid}${esc_c}" + if ! check_service_up "${pid}" 5000 250 ${http_port} ${grpc_port}; then + print_info "stderr:" + cat "${startup_error_tmp}" + rm "${startup_error_tmp}" + print_info "Please check ${LOG_STDERR} and ${LOG_STDOUT} for more details" + exit 1 + fi +} + +# --------------- Functions for stop--------------- +# Function to kill a process +kill_process() { + local pid=$1 + local signal=$2 + kill ${signal} "${pid}" 2>/dev/null +} + +# Stop service +stop_port() { + local port=$1 + local name=$2 + local pid=$(get_pid ${port}) + + print_info "Stopping $name ${highlight_c}${port}${esc_c}..." + if [ -n "${pid}" ]; then + for _ in {1..100}; do + sleep 0.1 + kill_process "${pid}" + pid=$(get_pid ${port}) + if [ -z "${pid}" ]; then + print_ok "Stop $name ${highlight_c}${port}${esc_c} success (SIGTERM)" + return + fi + done + kill_process "${pid}" -9 && print_ok "Stop port success (SIGKILL)" || print_error "Stop port failed" + else + print_ok "Stop $name ${highlight_c}${port}${esc_c} success(NOT ACTIVE)" + fi +} +stop() { + print_info "--------------------------------stopping--------------------------------" + stop_port ${http_port} "HTTP port" + stop_port ${grpc_port} "gRPC port" +} + +# --------------- Functions for status--------------- +# Check the status of the service +status() { + print_info "---------------------------------status---------------------------------" + # Check http_port + pid1=$(check_port_active $http_port) + if [ -n "${pid1}" ]; then + print_ok "Check http port ${highlight_c}${http_port}${esc_c} is active: PID=${highlight_c}${pid1}${esc_c}" + else + print_error "http port not active" + fi + + # Check grpc_port + pid2=$(check_port_active $grpc_port) + if [ -n "${pid2}" ]; then + print_ok "Check grpc port ${highlight_c}${grpc_port}${esc_c} is active: PID=${highlight_c}${pid2}${esc_c}" + else + print_error "grpc port not active" + fi + + # Check if both PIDs are the same + if [ -n "${pid1}" ] && [ -n "${pid2}" ]; then + if [ "${pid1}" == "${pid2}" ]; then + print_ok "Check http port and grpc port from same process: PID=${highlight_c}${pid2}${esc_c}" + else + print_error "Found http port and grpc port active but from different process: ${highlight_c}${pid2}${esc_c}!=${highlight_c}${pid2}${esc_c}" + fi + fi +} + +# --------------- Functions for info--------------- +# Print usage information for the script +print_usage() { + echo -e "${ok_c}FATE Flow${esc_c}" + echo "---------" + echo -e "${ok_c}Usage:${esc_c}" + echo -e " $0 start - Start the server application." + echo -e " $0 stop - Stop the server application." + echo -e " $0 status - Check and report the status of the server application." + echo -e " $0 restart [time] - Restart the server application. Optionally, specify a sleep time (in seconds) between stop and start." + echo "" + echo -e "${ok_c}Examples:${esc_c}" + echo " $0 start" + echo " $0 restart 5" + echo "" + echo -e "${ok_c}Notes:${esc_c}" + echo " - The restart command, if given an optional sleep time, will wait for the specified number of seconds between stopping and starting the service." + echo " If not provided, it defaults to 10 seconds." + echo " - Ensure that the required configuration file 'service_conf.yaml' is properly set up in the expected directory." + echo "" + echo "For more detailed information, refer to the script's documentation or visit the official documentation website." +} + +# --------------- Main--------------- +# Main case for control +case "$2" in + start) + load_config + start + status + ;; + starting) + load_config + start front + ;; + stop) + load_config + stop + ;; + status) + load_config + status + ;; + restart) + load_config + stop + sleep_time=${2:-5} + print_info "Waiting ${sleep_time} seconds" + sleep $sleep_time + start + status + ;; + *) + print_usage + exit 1 + ;; +esac diff --git a/python/fate_flow/component_env_utils/env_utils.py b/python/fate_flow/component_env_utils/env_utils.py deleted file mode 100644 index 33d80f9ce..000000000 --- a/python/fate_flow/component_env_utils/env_utils.py +++ /dev/null @@ -1,54 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import sys - -from fate_flow.component_env_utils.provider_utils import get_provider_class_object -from fate_flow.entity import ComponentProvider -from fate_flow.manager.provider_manager import ProviderManager - - -def get_python_path(provider_info): - return provider_info.get("env").get("PYTHONPATH") - - -def get_fate_algorithm_path(provider_info): - return provider_info.get("path") - - -def get_class_path(provider_info, name): - return provider_info.get("class_path").get(name) - - -def import_path(path): - sys.path.append(path) - - -def import_python_path(provider_info): - import_path(get_python_path(provider_info)) - - -def import_component_output_depend(provider_info=None): - if not provider_info: - provider_info = ProviderManager.get_default_fate_provider().to_dict() - import_python_path(provider_info) - - -def get_class_object(class_name): - provider_info = ProviderManager.get_default_fate_provider().to_dict() - import_python_path(provider_info) - provider = ComponentProvider(**provider_info) - class_obj = get_provider_class_object(provider, class_name) - return class_obj diff --git a/python/fate_flow/component_env_utils/feature_utils.py b/python/fate_flow/component_env_utils/feature_utils.py deleted file mode 100644 index 4c3a42a4d..000000000 --- a/python/fate_flow/component_env_utils/feature_utils.py +++ /dev/null @@ -1,88 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import numpy - - -def get_component_output_data_line(src_key, src_value, schema=None, all_extend_header=None): - if not all_extend_header: - for inst in ["inst_id", "label", "weight"]: - all_extend_header[inst] = "" - data_line = [src_key] - is_str = False - if hasattr(src_value, "is_instance"): - for inst in ["inst_id", "label", "weight"]: - if getattr(src_value, inst) is not None: - data_line.append(getattr(src_value, inst)) - if inst == "inst_id" and schema: - all_extend_header[inst] = schema.get("match_id_name") - else: - if not all_extend_header[inst]: - all_extend_header[inst] = inst - elif inst == "inst_id" and schema.get("match_id_name"): - data_line.append(None) - elif inst == "label" and schema.get("label_name"): - data_line.append(None) - data_line.extend(dataset_to_list(src_value.features)) - elif isinstance(src_value, str): - data_line.extend([value for value in src_value.split(',')]) - is_str = True - else: - data_line.extend(dataset_to_list(src_value)) - return data_line, is_str, all_extend_header - - -def generate_header(all_extend_header, schema): - extend_header = [] - for inst in ["inst_id", "label", "weight"]: - if all_extend_header.get(inst): - extend_header.append(all_extend_header[inst]) - if not all_extend_header.get(inst) and inst == "inst_id" and schema.get("match_id_name"): - extend_header.append(schema.get("match_id_name")) - if not all_extend_header.get(inst) and inst == "label" and schema.get("label_name"): - extend_header.append(inst) - return extend_header - - -def get_deserialize_value(src_value, id_delimiter): - extend_header = [] - if hasattr(src_value, "is_instance"): - v_list = [] - for inst in ["inst_id", "label", "weight"]: - if getattr(src_value, inst) is not None: - v_list.append(getattr(src_value, inst)) - extend_header.append(inst) - v_list.extend(dataset_to_list(src_value.features)) - v_list = list(map(str, v_list)) - deserialize_value = id_delimiter.join(v_list) - elif isinstance(src_value, str): - deserialize_value = src_value - else: - deserialize_value = id_delimiter.join(list(map(str, dataset_to_list(src_value)))) - return deserialize_value, extend_header - - -def dataset_to_list(src): - if isinstance(src, numpy.ndarray): - return src.tolist() - elif isinstance(src, list): - return src - elif hasattr(src, "is_sparse_vector"): - vector = [0] * src.get_shape() - for idx, v in src.get_all_data(): - vector[idx] = v - return vector - else: - return [src] \ No newline at end of file diff --git a/python/fate_flow/component_env_utils/provider_utils.py b/python/fate_flow/component_env_utils/provider_utils.py deleted file mode 100644 index fa813e6a4..000000000 --- a/python/fate_flow/component_env_utils/provider_utils.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import importlib -import pathlib - -from fate_flow.entity import ComponentProvider -from fate_flow.utils.log_utils import getLogger - -LOGGER = getLogger() - - -def get_provider_interface(provider: ComponentProvider): - obj = get_provider_class_object(provider, "interface") - for i in ('name', 'version', 'path'): - setattr(obj, f'provider_{i}', getattr(provider, i)) - return obj - - -def get_provider_model_paths(provider: ComponentProvider): - model_import_path = get_provider_class_import_path(provider, "model") - model_module_dir = pathlib.Path(provider.path).joinpath(*model_import_path[1:]) - return model_module_dir, model_import_path - - -def get_provider_class_object(provider: ComponentProvider, class_name, module=False): - class_path = get_provider_class_import_path(provider, class_name) - if module: - return importlib.import_module(".".join(class_path)) - else: - return getattr(importlib.import_module(".".join(class_path[:-1])), class_path[-1]) - - -def get_provider_class_import_path(provider: ComponentProvider, class_name): - return f"{pathlib.Path(provider.path).name}.{provider.class_path.get(class_name)}".split(".") diff --git a/python/fate_flow/components/__init__.py b/python/fate_flow/components/__init__.py index e69de29bb..ae946a49c 100644 --- a/python/fate_flow/components/__init__.py +++ b/python/fate_flow/components/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/components/__main__.py b/python/fate_flow/components/__main__.py new file mode 100644 index 000000000..6d61c0981 --- /dev/null +++ b/python/fate_flow/components/__main__.py @@ -0,0 +1,27 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +execute with python -m fate.components --config xxx +""" + +if __name__ == "__main__": + import click + from fate_flow.components.entrypoint.cli import component + + cli = click.Group() + cli.add_command(component) + cli(prog_name="python -m fate_flow.components") diff --git a/python/fate_flow/components/_base.py b/python/fate_flow/components/_base.py deleted file mode 100644 index d2882a310..000000000 --- a/python/fate_flow/components/_base.py +++ /dev/null @@ -1,289 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import abc -import typing - -from fate_flow.utils.log_utils import getLogger -from fate_flow.components.param_extract import ParamExtract -from fate_flow.scheduling_apps.client.tracker_client import TrackerClient - - -LOGGER = getLogger() - - -class ComponentInputProtocol(metaclass=abc.ABCMeta): - - @property - @abc.abstractmethod - def parameters(self) -> dict: - ... - - @property - @abc.abstractmethod - def flow_feeded_parameters(self) -> dict: - ... - - @property - @abc.abstractmethod - def roles(self): - ... - - @property - @abc.abstractmethod - def job_parameters(self): - ... - - @property - @abc.abstractmethod - def tracker(self): - ... - - @property - @abc.abstractmethod - def task_version_id(self): - ... - - @property - @abc.abstractmethod - def checkpoint_manager(self): - ... - - @property - @abc.abstractmethod - def datasets(self): - ... - - @property - @abc.abstractmethod - def models(self): - ... - - -class ComponentOutput: - def __init__(self, data, models, cache: typing.List[tuple], serialize: bool = True) -> None: - self._data = data - if not isinstance(self._data, list): - self._data = [data] - - self._models = models - if self._models is None: - self._models = {} - - self._cache = cache - if not isinstance(self._cache, list): - self._cache = [cache] - - self.serialize = serialize - - @property - def data(self): - return self._data - - @property - def model(self): - if not self.serialize: - return self._models - - serialized_models: typing.Dict[str, typing.Tuple[str, bytes]] = {} - - for model_name, buffer_object in self._models.items(): - serialized_string = buffer_object.SerializeToString() - if not serialized_string: - from fate_arch.protobuf.python import default_empty_fill_pb2 - - buffer_object = default_empty_fill_pb2.DefaultEmptyFillMessage() - buffer_object.flag = "set" - serialized_string = buffer_object.SerializeToString() - pb_name = type(buffer_object).__name__ - serialized_models[model_name] = (pb_name, serialized_string) - - return serialized_models - - @property - def cache(self): - return self._cache - - -class ComponentBase(metaclass=abc.ABCMeta): - - def __init__(self): - self.task_version_id = "" - self.tracker: TrackerClient = None - self.checkpoint_manager = None - self.model_output = None - self.data_output = None - self.cache_output = None - self.serialize = True - - @abc.abstractmethod - def _run(self, cpn_input: ComponentInputProtocol): - """to be implemented""" - ... - - def _retry(self, cpn_input: ComponentInputProtocol): - ... - # raise NotImplementedError(f"_retry for {type(self)} not implemented") - - def run(self, cpn_input: ComponentInputProtocol, retry: bool = True): - self.task_version_id = cpn_input.task_version_id - self.tracker = cpn_input.tracker - self.checkpoint_manager = cpn_input.checkpoint_manager - - # retry - if ( - retry - and hasattr(self, '_retry') - and callable(self._retry) - and self.checkpoint_manager is not None - and self.checkpoint_manager.latest_checkpoint is not None - ): - self._retry(cpn_input=cpn_input) - # normal - else: - self._run(cpn_input=cpn_input) - - return ComponentOutput(data=self.save_data(), models=self.export_model(), cache=self.save_cache(), serialize=self.serialize) - - def save_data(self): - return self.data_output - - def export_model(self): - return self.model_output - - def save_cache(self): - return self.cache_output - - -class _RunnerDecorator: - def __init__(self, meta) -> None: - self._roles = set() - self._meta = meta - - @property - def on_guest(self): - self._roles.add("guest") - return self - - @property - def on_host(self): - self._roles.add("host") - return self - - @property - def on_arbiter(self): - self._roles.add("arbiter") - return self - - @property - def on_local(self): - self._roles.add("local") - return self - - def __call__(self, cls): - if issubclass(cls, ComponentBase): - for role in self._roles: - self._meta._role_to_runner_cls[role] = cls - else: - raise NotImplementedError(f"type of {cls} not supported") - - return cls - - -class ComponentMeta: - __name_to_obj: typing.Dict[str, "ComponentMeta"] = {} - - def __init__(self, name) -> None: - self.name = name - self._role_to_runner_cls = {} - self._param_cls = None - - self.__name_to_obj[name] = self - - @property - def bind_runner(self): - return _RunnerDecorator(self) - - @property - def bind_param(self): - def _wrap(cls): - self._param_cls = cls - return cls - - return _wrap - - def register_info(self): - return { - self.name: dict( - module=self.__module__, - ) - } - - @classmethod - def get_meta(cls, name): - return cls.__name_to_obj[name] - - def _get_runner(self, role: str): - if role not in self._role_to_runner_cls: - raise ModuleNotFoundError( - f"Runner for component `{self.name}` at role `{role}` not found" - ) - return self._role_to_runner_cls[role] - - def get_run_obj(self, role: str): - return self._get_runner(role)() - - def get_run_obj_name(self, role: str) -> str: - return self._get_runner(role).__name__ - - def get_param_obj(self, cpn_name: str): - if self._param_cls is None: - raise ModuleNotFoundError(f"Param for component `{self.name}` not found") - param_obj = self._param_cls().set_name(f"{self.name}#{cpn_name}") - return param_obj - - def get_supported_roles(self): - roles = set(self._role_to_runner_cls.keys()) - if not roles: - raise ModuleNotFoundError(f"roles for {self.name} is empty") - return roles - - -class BaseParam(object): - - def set_name(self, name: str): - self._name = name - return self - - def check(self): - raise NotImplementedError("Parameter Object should have be check") - - def as_dict(self): - return ParamExtract().change_param_to_dict(self) - - @classmethod - def from_dict(cls, conf): - obj = cls() - obj.update(conf) - return obj - - def update(self, conf, allow_redundant=False): - return ParamExtract().recursive_parse_param_from_config( - param=self, - config_json=conf, - param_parse_depth=0, - valid_check=not allow_redundant, - name=self._name, - ) diff --git a/python/fate_flow/components/api_reader.py b/python/fate_flow/components/api_reader.py deleted file mode 100644 index 4abe581e2..000000000 --- a/python/fate_flow/components/api_reader.py +++ /dev/null @@ -1,228 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import json -import os -import time -from contextlib import closing - -import requests -from requests_toolbelt import MultipartEncoder - -from fate_arch.common.data_utils import default_output_info -from fate_arch.session import Session -from fate_flow.components._base import ComponentMeta, BaseParam, ComponentBase, ComponentInputProtocol -from fate_flow.db.service_registry import ServiceRegistry -from fate_flow.entity import Metric -from fate_flow.settings import TEMP_DIRECTORY -from fate_flow.utils.data_utils import convert_output -from fate_flow.utils.log_utils import getLogger -from fate_flow.utils.upload_utils import UploadFile - -logger = getLogger() -api_reader_cpn_meta = ComponentMeta("ApiReader") - - -@api_reader_cpn_meta.bind_param -class ApiReaderParam(BaseParam): - def __init__( - self, - server_name=None, - parameters=None, - id_delimiter=",", - head=True, - extend_sid=False, - timeout=60 * 12 - ): - self.server_name = server_name - self.parameters = parameters - self.id_delimiter = id_delimiter - self.head = head - self.extend_sid = extend_sid - self.timeout = timeout - - def check(self): - return True - - -@api_reader_cpn_meta.bind_runner.on_guest.on_host -class ApiReader(ComponentBase): - def __init__(self): - super(ApiReader, self).__init__() - self.parameters = {} - self.required_url_key_list = ["upload", "query", "download"] - self.service_info = {} - - def _run(self, cpn_input: ComponentInputProtocol): - self.cpn_input = cpn_input - self.parameters = cpn_input.parameters - self.task_dir = os.path.join(TEMP_DIRECTORY, self.tracker.task_id, str(self.tracker.task_version)) - for cpn_name, data in cpn_input.datasets.items(): - for data_name, table_list in data.items(): - self.input_table = table_list[0] - logger.info(f"parameters: {self.parameters}") - if not self.parameters.get("server_name"): - self._run_guest() - else: - self._run_host() - - def _run_guest(self): - self.data_output = [self.input_table] - - def _run_host(self): - self.set_service_registry_info() - response = self.upload_data() - logger.info(f"upload response: {response.text}") - if response.status_code == 200: - response_data = response.json() - if response_data.get("code") == 0: - logger.info(f"request success, start check status") - job_id = response_data.get("data").get("jobId") - status = self.check_status(job_id) - if status: - download_path = self.download_data(job_id) - table, output_name, output_namespace = self.output_feature_table() - count = UploadFile.upload( - download_path, - head=self.parameters.get("head"), - table=table, - id_delimiter=self.parameters.get("id_delimiter"), - extend_sid=self.parameters.get("extend_sid") - ) - table.meta.update_metas(count=count) - self.tracker.log_output_data_info( - data_name=self.cpn_input.flow_feeded_parameters.get("output_data_name")[0], - table_namespace=output_namespace, - table_name=output_name, - ) - self.tracker.log_metric_data( - metric_namespace="api_reader", - metric_name="upload", - metrics=[Metric("count", count)], - ) - else: - raise Exception(f"upload return: {response.text}") - - def output_feature_table(self): - ( - output_name, - output_namespace - ) = default_output_info( - task_id=self.tracker.task_id, - task_version=self.tracker.task_version, - output_type="data" - ) - logger.info(f"flow_feeded_parameters: {self.cpn_input.flow_feeded_parameters}") - input_table_info = self.cpn_input.flow_feeded_parameters.get("table_info")[0] - _, output_table_address, output_table_engine = convert_output( - input_table_info["name"], - input_table_info["namespace"], - output_name, - output_namespace, self.input_table.engine - ) - sess = Session.get_global() - output_table_session = sess.storage(storage_engine=output_table_engine) - table = output_table_session.create_table( - address=output_table_address, - name=output_name, - namespace=output_namespace, - partitions=self.input_table.partitions, - ) - return table, output_name, output_namespace - - def check_status(self, job_id): - query_registry_info = self.service_info.get("query") - logger.info(f"parameters timeout: {self.parameters.get('timeout', 60 * 12)} min") - for i in range(0, self.parameters.get("timeout", 60 * 12)): - status_response = getattr(requests, query_registry_info.f_method.lower(), None)( - url=query_registry_info.f_url, - json={"jobId": job_id} - ) - logger.info(f"status: {status_response.text}") - if status_response.status_code == 200: - if status_response.json().get("data").get("status").lower() == "success": - logger.info(f"job id {job_id} status success, start download") - return True - if status_response.json().get("data").get("status").lower() != "running": - logger.error(f"job id {job_id} status: {status_response.json().get('data').get('status')}") - raise Exception(status_response.json().get("data")) - logger.info(f"job id {job_id} status: {status_response.json().get('data').get('status')}") - time.sleep(60) - raise TimeoutError("check status timeout") - - def download_data(self, job_id): - download_registry_info = self.service_info.get("download") - download_path = os.path.join(self.task_dir, "features") - logger.info(f"start download feature, url: {download_registry_info.f_url}") - params = {"jobId": job_id} - en_content = self.encrypt_content(job_id) - if en_content: - params.update({"sign": en_content}) - with closing(getattr(requests, download_registry_info.f_method.lower(), None)( - url=download_registry_info.f_url, - params={"requestBody": json.dumps(params)}, - stream=True)) as response: - if response.status_code == 200: - with open(download_path, 'wb') as fw: - for chunk in response.iter_content(1024): - if chunk: - fw.write(chunk) - else: - raise Exception(f"download return: {response.text}") - return download_path - - def upload_data(self): - id_path = os.path.join(self.task_dir, "id") - logger.info(f"save to: {id_path}") - os.makedirs(os.path.dirname(id_path), exist_ok=True) - with open(id_path, "w") as f: - if self.input_table.schema: - id_name = self.input_table.schema.get("sid_name") or self.input_table.schema.get("sid", "sid") - f.write(f"{id_name}\n") - for k, _ in self.input_table.collect(): - f.write(f"{k}\n") - with open(id_path, "rb") as f: - data = MultipartEncoder( - fields={'file': ("id", f, 'application/octet-stream')} - ) - upload_registry_info = self.service_info.get("upload") - logger.info(f"upload info:{upload_registry_info.to_dict()}") - params = self.parameters.get("parameters", {}) - params.update({"job_id": self.tracker.job_id, }) - en_content = self.encrypt_content() - if en_content: - params.update({"sign": en_content}) - response = getattr(requests, upload_registry_info.f_method.lower(), None)( - url=upload_registry_info.f_url, - params={"requestBody": json.dumps(params)}, - data=data, - headers={'Content-Type': data.content_type} - ) - return response - - def set_service_registry_info(self): - for info in ServiceRegistry().load_service(server_name=self.parameters.get("server_name")): - for key in self.required_url_key_list: - if key == info.f_service_name: - self.service_info[key] = info - logger.info(f"set service registry info:{self.service_info}") - - def encrypt_content(self, job_id=None): - if not job_id: - job_id = self.tracker.job_id - import hashlib - md5 = hashlib.md5() - md5.update(job_id.encode()) - return md5.hexdigest() diff --git a/python/fate_flow/components/cache_loader.py b/python/fate_flow/components/cache_loader.py deleted file mode 100644 index 7211d5d56..000000000 --- a/python/fate_flow/components/cache_loader.py +++ /dev/null @@ -1,79 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_flow.utils.log_utils import getLogger -from fate_flow.components._base import ( - BaseParam, - ComponentBase, - ComponentMeta, - ComponentInputProtocol, -) -from fate_flow.operation.job_tracker import Tracker -from fate_flow.entity import MetricMeta, MetricType - -LOGGER = getLogger() - -cache_loader_cpn_meta = ComponentMeta("CacheLoader") - - -@cache_loader_cpn_meta.bind_param -class CacheLoaderParam(BaseParam): - def __init__(self, cache_key=None, job_id=None, component_name=None, cache_name=None): - super().__init__() - self.cache_key = cache_key - self.job_id = job_id - self.component_name = component_name - self.cache_name = cache_name - - def check(self): - return True - - -@cache_loader_cpn_meta.bind_runner.on_guest.on_host -class CacheLoader(ComponentBase): - def __init__(self): - super(CacheLoader, self).__init__() - self.parameters = {} - self.cache_key = None - self.job_id = None - self.component_name = None - self.cache_name = None - - def _run(self, cpn_input: ComponentInputProtocol): - self.parameters = cpn_input.parameters - LOGGER.info(self.parameters) - for k, v in self.parameters.items(): - if hasattr(self, k): - setattr(self, k, v) - tracker = Tracker(job_id=self.job_id, - role=self.tracker.role, - party_id=self.tracker.party_id, - component_name=self.component_name) - LOGGER.info(f"query cache by cache key: {self.cache_key} cache name: {self.cache_name}") - # todo: use tracker client but not tracker - caches = tracker.query_output_cache(cache_key=self.cache_key, cache_name=self.cache_name) - if not caches: - raise Exception("can not found this cache") - elif len(caches) > 1: - raise Exception(f"found {len(caches)} caches, only support one, please check parameters") - else: - cache = caches[0] - self.cache_output = cache - tracker.job_id = self.tracker.job_id - tracker.component_name = self.tracker.component_name - metric_meta = cache.to_dict() - metric_meta.pop("data") - metric_meta["component_name"] = self.component_name - self.tracker.set_metric_meta(metric_namespace="cache_loader", metric_name=cache.name, metric_meta=MetricMeta(name="cache", metric_type=MetricType.CACHE_INFO, extra_metas=metric_meta)) diff --git a/python/fate_flow/components/components.py b/python/fate_flow/components/components.py deleted file mode 100644 index 78e89ae83..000000000 --- a/python/fate_flow/components/components.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import importlib -import inspect -import typing -from pathlib import Path - -from fate_flow.utils.log_utils import getLogger -from fate_flow.components._base import ComponentMeta - - -LOGGER = getLogger() - - -def _get_module_name_by_path(path, base): - return '.'.join(path.resolve().relative_to(base.resolve()).with_suffix('').parts) - - -def _search_components(path, base): - try: - module_name = _get_module_name_by_path(path, base) - module = importlib.import_module(module_name) - except ImportError as e: - # or skip ? - raise e - _obj_pairs = inspect.getmembers(module, lambda obj: isinstance(obj, ComponentMeta)) - return _obj_pairs, module_name - - -class Components: - provider_version = None - provider_name = None - provider_path = None - - @classmethod - def _module_base(cls): - return Path(cls.provider_path).resolve().parent - - @classmethod - def _components_base(cls): - return Path(cls.provider_path, 'components').resolve() - - @classmethod - def get_names(cls) -> typing.Dict[str, dict]: - names = {} - for p in cls._components_base().glob("**/*.py"): - obj_pairs, module_name = _search_components(p.resolve(), cls._module_base()) - for name, obj in obj_pairs: - names[obj.name] = {"module": module_name} - LOGGER.info(f"component register {obj.name} with cache info {module_name}") - return names - - @classmethod - def get(cls, name: str, cache) -> ComponentMeta: - if cache: - importlib.import_module(cache[name]["module"]) - else: - for p in cls._components_base().glob("**/*.py"): - module_name = _get_module_name_by_path(p, cls._module_base()) - importlib.import_module(module_name) - - cpn = ComponentMeta.get_meta(name) - return cpn diff --git a/python/fate_flow/components/components/__init__.py b/python/fate_flow/components/components/__init__.py new file mode 100644 index 000000000..64524f308 --- /dev/null +++ b/python/fate_flow/components/components/__init__.py @@ -0,0 +1,18 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .upload import upload +from .download import download + +BUILDIN_COMPONENTS = [upload, download] diff --git a/python/fate_flow/components/components/download.py b/python/fate_flow/components/components/download.py new file mode 100644 index 000000000..6b921fc6c --- /dev/null +++ b/python/fate_flow/components/components/download.py @@ -0,0 +1,76 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from fate_flow.components import cpn +from fate_flow.engine import storage +from fate_flow.entity.spec.dag import IOMeta, ArtifactOutputSpec, Metadata, TaskConfigSpec +from fate_flow.errors.server_error import NoFoundTable +from fate_flow.manager.outputs.data import DataManager + + +@cpn.component() +def download( + config, + outputs: IOMeta.OutputMeta +): + download_data(config, outputs) + + +def download_data(config: TaskConfigSpec, outputs: IOMeta.OutputMeta): + parameters = config.parameters + download_object = Download() + download_object.run( + parameters=DownloadParam( + **parameters + ), + outputs=outputs + ) + + +class DownloadParam(object): + def __init__( + self, + namespace, + name, + path, + ): + self.name = name + self.namespace = namespace + self.path = path + + +class Download: + def __init__(self): + self.parameters = None + self.table = None + self.data_meta = {} + + def run(self, parameters: DownloadParam, outputs: IOMeta.OutputMeta): + data_table_meta = storage.StorageTableMeta(name=parameters.name, namespace=parameters.namespace) + if not data_table_meta: + raise NoFoundTable(name=parameters.name, namespace=parameters.namespace) + download_dir = parameters.path + logging.info("start download data") + DataManager.send_table( + output_tables_meta={"data": data_table_meta}, + download_dir=download_dir + ) + outputs.data = {"output_data": ArtifactOutputSpec( + uri="", + metadata=Metadata(namespace=parameters.namespace, name=parameters.name), + type_name=data_table_meta.data_type, + ).dict()} + logging.info(f"download data success, download path: {parameters.path}") diff --git a/python/fate_flow/components/components/upload.py b/python/fate_flow/components/components/upload.py new file mode 100644 index 000000000..bd02a366b --- /dev/null +++ b/python/fate_flow/components/components/upload.py @@ -0,0 +1,385 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import logging +import os +import secrets +import uuid +from typing import Union + +from fate_flow.components import cpn +from fate_flow.engine.storage import Session, StorageEngine, DataType, StorageTableMeta +from fate_flow.entity.spec.dag import IOMeta, ArtifactOutputSpec, Metadata, ArtifactSource, MetricData, TaskConfigSpec +from fate_flow.entity.types import JsonMetricArtifactType, EngineType +from fate_flow.manager.outputs.data import DatasetManager +from fate_flow.runtime.system_settings import STANDALONE_DATA_HOME, ENGINES +from fate_flow.utils.file_utils import get_fate_flow_directory +from fate_flow.utils.io_utils import URI + + +@cpn.component() +def upload( + config, outputs: IOMeta.OutputMeta +): + upload_data(config, outputs) + + +def upload_data(config: TaskConfigSpec, outputs): + parameters = config.parameters + job_id = config.job_id + upload_object = Upload() + engine_options = {} + if config.conf.computing.metadata and config.conf.computing.metadata.options: + engine_options = config.conf.computing.metadata.options + data = upload_object.run( + parameters=UploadParam( + **parameters + ), + job_id=job_id, + outputs=outputs, + engine_options=engine_options + ) + + +class Param(object): + def to_dict(self): + d = {} + for k, v in self.__dict__.items(): + if v is None: + continue + d[k] = v + return d + + +class MetaParam(Param): + def __init__( + self, + sample_id_name: str = None, + match_id_name: str = None, + match_id_list: list = None, + match_id_range: int = 0, + label_name: Union[None, str] = None, + label_type: str = "int32", + weight_name: Union[None, str] = None, + weight_type: str = "float32", + header: str = None, + delimiter: str = ",", + dtype: Union[str, dict] = "float32", + na_values: Union[str, list, dict] = None, + input_format: str = "dense", + tag_with_value: bool = False, + tag_value_delimiter: str = ":" + ): + self.sample_id_name = sample_id_name + self.match_id_name = match_id_name + self.match_id_list = match_id_list + self.match_id_range = match_id_range + self.label_name = label_name + self.label_type = label_type + self.weight_name = weight_name + self.weight_type = weight_type + self.header = header + self.delimiter = delimiter + self.dtype = dtype + self.na_values = na_values + self.input_format = input_format + self.tag_with_value = tag_with_value + self.tag_value_delimiter = tag_value_delimiter + + +class UploadParam(Param): + def __init__( + self, + namespace="", + name="", + file="", + storage_engine="", + head=1, + partitions=10, + extend_sid=False, + is_temp_file=False, + address: dict = {}, + meta: dict = {} + ): + self.name = name + self.namespace = namespace + self.file = file + self.storage_engine = storage_engine + self.head = head + self.partitions = partitions + self.extend_sid = extend_sid + self.meta = MetaParam(**meta) + self.storage_address = address + self.is_temp_file = is_temp_file + + +class Upload: + def __init__(self): + self.parameters = None + self.table = None + self.data_meta = {} + + def run(self, parameters: UploadParam, outputs: IOMeta.OutputMeta = None, job_id="", engine_options={}): + self.parameters = parameters + logging.info(self.parameters.to_dict()) + storage_address = self.parameters.storage_address + if not os.path.isabs(parameters.file): + parameters.file = os.path.join( + get_fate_flow_directory(), parameters.file + ) + name, namespace = parameters.name, parameters.namespace + if not name or not namespace: + namespace, name = self.parameters.namespace, self.parameters.name = self.generate_table_name() + if not parameters.storage_engine: + parameters.storage_engine = ENGINES.get(EngineType.STORAGE) + with Session() as sess: + # clean table + table = sess.get_table(namespace=namespace, name=name) + if table: + logging.info( + f"destroy table name: {name} namespace: {namespace} engine: {table.engine}" + ) + try: + table.destroy() + except Exception as e: + logging.error(e) + else: + logging.info( + f"can not found table name: {name} namespace: {namespace}, pass destroy" + ) + address_dict = storage_address.copy() + storage_engine = self.parameters.storage_engine + storage_session = sess.storage( + storage_engine=storage_engine, options=engine_options + ) + if storage_engine in {StorageEngine.EGGROLL, StorageEngine.STANDALONE}: + upload_address = { + "name": name, + "namespace": namespace + } + if storage_engine == StorageEngine.STANDALONE: + home = os.getenv("STANDALONE_DATA_HOME") or STANDALONE_DATA_HOME + upload_address.update({"home": home}) + elif storage_engine in {StorageEngine.HDFS, StorageEngine.FILE}: + upload_address = { + "path": DatasetManager.upload_data_path( + name=name, + namespace=namespace, + storage_engine=storage_engine + ) + } + else: + raise RuntimeError(f"can not support this storage engine: {storage_engine}") + address_dict.update(upload_address) + logging.info(f"upload to {storage_engine} storage, address: {address_dict}") + logging.info(f"engine options: {engine_options}") + address = StorageTableMeta.create_address( + storage_engine=storage_engine, address_dict=address_dict + ) + self.table = storage_session.create_table( + address=address, + source=ArtifactSource( + task_id="", + party_task_id="", + task_name="upload", + component="upload", + output_artifact_key="data" + ).dict(), + **self.parameters.to_dict(), + options=engine_options + ) + data_table_count = self.save_data_table(job_id) + logging.info("------------load data finish!-----------------") + + logging.info("file: {}".format(self.parameters.file)) + logging.info("total data_count: {}".format(data_table_count)) + logging.info("table name: {}, table namespace: {}".format(name, namespace)) + if outputs: + self.save_outputs(job_id, outputs, data_table_count) + return {"name": name, "namespace": namespace, "count": data_table_count, "data_meta": self.data_meta} + + def save_data_table(self, job_id): + input_file = self.parameters.file + input_feature_count = self.get_count(input_file) + self.upload_file(input_file, job_id, input_feature_count) + table_count = self.table.count() + metas_info = { + "count": table_count, + "partitions": self.parameters.partitions, + "data_type": DataType.TABLE + } + self.table.meta.update_metas(**metas_info) + # cleanup temp file + self.cleanup() + return table_count + + def update_schema(self, fp): + id_index = 0 + read_status = False + if self.parameters.head is True: + data_head = fp.readline() + id_index = self.update_table_meta(data_head) + read_status = True + else: + pass + return id_index, read_status + + def upload_file(self, input_file, job_id, input_feature_count=None, table=None): + if not table: + table = self.table + part_of_data = [] + with open(input_file, "r") as fp: + id_index, read_status = self.update_schema(fp) + if read_status: + input_feature_count -= 1 + self.table.put_all(self.kv_generator(input_feature_count, fp, job_id, part_of_data, id_index=id_index)) + table.meta.update_metas(part_of_data=part_of_data) + + def get_line(self): + if not self.parameters.extend_sid: + line = self.get_data_line + else: + line = self.get_sid_data_line + return line + + @staticmethod + def get_data_line(values, delimiter, id_index, **kwargs): + if id_index: + k = values[id_index] + v = delimiter.join([ + delimiter.join(values[:id_index]), + delimiter.join(values[id_index + 1:]) + ]).strip(delimiter) + else: + k = values[0] + v = delimiter.join(list(map(str, values[1:]))) + return k, v + + @staticmethod + def get_sid_data_line(values, delimiter, fate_uuid, line_index, **kwargs): + return fate_uuid + str(line_index), delimiter.join(list(map(str, values[:]))) + + def kv_generator(self, input_feature_count, fp, job_id, part_of_data, id_index): + fate_uuid = secrets.token_bytes(16).hex() + get_line = self.get_line() + line_index = 0 + logging.info(input_feature_count) + while True: + lines = fp.readlines(104857600) + if lines: + for line in lines: + values = line.rstrip().split(self.parameters.meta.delimiter) + k, v = get_line( + values=values, + line_index=line_index, + delimiter=self.parameters.meta.delimiter, + fate_uuid=fate_uuid, + id_index=id_index + ) + yield k, v + line_index += 1 + if line_index <= 100: + part_of_data.append((k, v)) + save_progress = line_index / input_feature_count * 100 // 1 + job_info = { + "progress": save_progress, + "job_id": job_id, + "role": "local", + "party_id": 0, + } + logging.info(f"job info: {job_info}") + else: + return + + def get_count(self, input_file): + with open(input_file, "r", encoding="utf-8") as fp: + count = 0 + for line in fp: + count += 1 + return count + + def update_table_meta(self, data_head): + logging.info(f"data head: {data_head}") + update_schema, id_index = self.get_header_schema( + header_line=data_head + ) + self.data_meta.update(self.parameters.meta.to_dict()) + self.data_meta.update(update_schema) + self.table.meta.update_metas(data_meta=self.data_meta) + return id_index + + def get_header_schema(self, header_line): + delimiter = self.parameters.meta.delimiter + sample_id_name = self.parameters.meta.sample_id_name + sample_id_index = 0 + if self.parameters.extend_sid: + sample_id_name = "extend_sid" + header = delimiter.join([sample_id_name, header_line]).strip() + else: + header_list = header_line.split(delimiter) + if not sample_id_name: + # default set sample_id_index = 0 + sample_id_name = header_list[0] + else: + if sample_id_name not in header_line: + raise RuntimeError(f"No found sample id {sample_id_name} in header") + sample_id_index = header_list.index(sample_id_name) + if sample_id_index > 0: + header_line = self.join_in_index_line(delimiter, header_list, sample_id_index) + header = header_line.strip() + return {'header': header, "sample_id_name": sample_id_name}, sample_id_index + + @staticmethod + def join_in_index_line(delimiter, values, id_index): + return delimiter.join([ + values[id_index], + delimiter.join(values[:id_index]), + delimiter.join(values[id_index + 1:]) + ]).strip(delimiter) + + def save_outputs(self, job_id, outputs: IOMeta.OutputMeta, data_count): + data = ArtifactOutputSpec( + uri="", + metadata=Metadata(namespace=self.parameters.namespace, name=self.parameters.name), + type_name=DataType.TABLE, + ) + uri = DatasetManager.output_local_uri( + task_info=dict(job_id=job_id, role="local", party_id="0", task_name="upload_0", task_version="0"), + name="metric", + type_name=JsonMetricArtifactType.type_name + ) + + path = URI.from_string(uri).to_schema().path + os.makedirs(os.path.dirname(path), exist_ok=True) + metrics = [MetricData(name="upload", data={"name": self.parameters.name, "namespace": self.parameters.namespace, + "count": data_count}).dict()] + with open(path, "w") as f: + json.dump(metrics, f) + metric = ArtifactOutputSpec( + uri=uri, + metadata=Metadata(metadata={}), + type_name=JsonMetricArtifactType.type_name + ) + outputs.data = {"table": data.dict()} + outputs.metric = {"metric": metric.dict()} + + @staticmethod + def generate_table_name(): + return "upload", uuid.uuid1().hex + + def cleanup(self): + if self.parameters.is_temp_file: + if os.path.exists(self.parameters.file): + os.remove(self.parameters.file) diff --git a/python/fate_flow/components/cpn.py b/python/fate_flow/components/cpn.py new file mode 100644 index 000000000..a19778842 --- /dev/null +++ b/python/fate_flow/components/cpn.py @@ -0,0 +1,55 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import inspect +import logging +from typing import Any + +from pydantic import BaseModel + + +class Params(BaseModel): + class TaskParams(BaseModel): + job_id: str + + component_params: Any + task_params: TaskParams + + +class _Component: + def __init__( + self, + name: str, + callback + ) -> None: + self.name = name + self.callback = callback + + def execute(self, config, outputs): + return self.callback(config, outputs) + + +def component(*args, **kwargs): + def decorator(f): + cpn_name = f.__name__.lower() + if isinstance(f, _Component): + raise TypeError("Attempted to convert a callback into a component twice.") + cpn = _Component( + name=cpn_name, + callback=f + ) + cpn.__doc__ = f.__doc__ + # cpn.validate_declare() + return cpn + return decorator diff --git a/python/fate_flow/components/define/download.yaml b/python/fate_flow/components/define/download.yaml new file mode 100644 index 000000000..63fccb1a5 --- /dev/null +++ b/python/fate_flow/components/define/download.yaml @@ -0,0 +1,61 @@ +component: + name: download + description: '' + provider: fate_flow + version: 2.0.0 + labels: [] + roles: + - guest + - host + - local + parameters: + name: + type: str + default: + optional: false + description: '' + type_meta: + title: str + type: string + default: + description: '' + namespace: + type: str + default: + optional: false + description: '' + type_meta: + title: str + type: string + default: + description: '' + path: + type: str + default: + optional: false + description: '' + type_meta: + title: str + type: string + default: + description: '' + input_artifacts: + data: {} + model: {} + output_artifacts: + data: {} + model: {} + metric: + metric: + types: + - json_metric + optional: false + stages: + - default + roles: + - guest + - host + - local + description: metric, invisible for user + is_multi: false +schema_version: v1 diff --git a/python/fate_flow/components/define/upload.yaml b/python/fate_flow/components/define/upload.yaml new file mode 100644 index 000000000..4027ef457 --- /dev/null +++ b/python/fate_flow/components/define/upload.yaml @@ -0,0 +1,143 @@ +component: + name: upload + description: '' + provider: fate_flow + version: 2.0.0 + labels: [] + roles: + - guest + - host + - local + parameters: + name: + type: str + default: + optional: false + description: '' + type_meta: + title: str + type: string + default: + description: '' + namespace: + type: str + default: + optional: false + description: '' + type_meta: + title: str + type: string + default: + description: '' + file: + type: str + default: + optional: false + description: '' + type_meta: + title: str + type: string + default: + description: '' + storage_engine: + type: str + default: + optional: true + description: '' + type_meta: + title: str + type: string + default: + description: '' + head: + type: str + default: + optional: true + description: '' + type_meta: + title: str + type: string + default: + description: '' + partitions: + type: str + default: + optional: true + description: '' + type_meta: + title: str + type: string + default: + description: '' + extend_sid: + type: str + default: + optional: true + description: '' + type_meta: + title: str + type: string + default: + description: '' + is_temp_file: + type: str + default: + optional: true + description: '' + type_meta: + title: str + type: string + default: + description: '' + address: + type: str + default: + optional: true + description: '' + type_meta: + title: str + type: string + default: + description: '' + meta: + type: str + default: + optional: true + description: '' + type_meta: + title: str + type: string + default: + description: '' + input_artifacts: + data: {} + model: {} + output_artifacts: + data: + table: + types: + - table + optional: false + stages: + - default + roles: + - guest + - host + - local + description: '' + is_multi: false + model: {} + metric: + metric: + types: + - json_metric + optional: false + stages: + - default + roles: + - guest + - host + - local + description: metric, invisible for user + is_multi: false +schema_version: v1 diff --git a/python/fate_flow/components/download.py b/python/fate_flow/components/download.py deleted file mode 100644 index beeffa749..000000000 --- a/python/fate_flow/components/download.py +++ /dev/null @@ -1,93 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os - -from fate_arch import storage -from fate_flow.manager.data_manager import TableStorage -from fate_flow.utils.log_utils import getLogger -from fate_arch.storage import DEFAULT_ID_DELIMITER -from fate_flow.components._base import ( - BaseParam, - ComponentBase, - ComponentMeta, - ComponentInputProtocol, -) -from fate_flow.entity import Metric, MetricMeta - -LOGGER = getLogger() - -download_cpn_meta = ComponentMeta("Download") - - -@download_cpn_meta.bind_param -class DownloadParam(BaseParam): - def __init__( - self, - output_path="", - delimiter=DEFAULT_ID_DELIMITER, - namespace="", - name="", - ): - self.output_path = output_path - self.delimiter = delimiter - self.namespace = namespace - self.name = name - - def check(self): - return True - - -@download_cpn_meta.bind_runner.on_local -class Download(ComponentBase): - def __init__(self): - super(Download, self).__init__() - self.parameters = {} - - def _run(self, cpn_input: ComponentInputProtocol): - self.parameters = cpn_input.parameters - self.parameters["role"] = cpn_input.roles["role"] - self.parameters["local"] = cpn_input.roles["local"] - - data_table_meta = storage.StorageTableMeta(name=self.parameters.get("name"), namespace=self.parameters.get("namespace")) - TableStorage.send_table( - output_tables_meta={"table": data_table_meta}, - output_data_file_path = os.path.abspath(self.parameters["output_path"]), - local_download=True - ) - self.callback_metric( - metric_name="data_access", - metric_namespace="download", - metric_data=[Metric("count", data_table_meta.count)] - ) - LOGGER.info("===== export {} lines totally =====".format(data_table_meta.count)) - LOGGER.info("===== export data finish =====") - LOGGER.info( - "===== export data file path:{} =====".format( - os.path.abspath(self.parameters["output_path"]) - ) - ) - - def callback_metric(self, metric_name, metric_namespace, metric_data): - self.tracker.log_metric_data( - metric_name=metric_name, - metric_namespace=metric_namespace, - metrics=metric_data, - ) - self.tracker.set_metric_meta( - metric_namespace, - metric_name, - MetricMeta(name="download", metric_type="DOWNLOAD"), - ) diff --git a/python/fate_flow/components/entrypoint/__init__.py b/python/fate_flow/components/entrypoint/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/components/entrypoint/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/components/entrypoint/cli.py b/python/fate_flow/components/entrypoint/cli.py new file mode 100644 index 000000000..196669b25 --- /dev/null +++ b/python/fate_flow/components/entrypoint/cli.py @@ -0,0 +1,130 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import logging +import os +import traceback + +import click + +from fate_flow.entity.spec.dag import PreTaskConfigSpec, TaskConfigSpec, IOMeta +from fate_flow.hub.flow_hub import FlowHub + +logger = logging.getLogger(__name__) + + +@click.group() +def component(): + """ + Manipulate components: execute, list, generate describe file + """ + + +@component.command() +@click.option("--config", required=False, type=click.File(), help="config path") +@click.option("--env-name", required=False, type=str, help="env name for config") +@click.option("--wraps-module", required=False, type=str, help="component run wraps module") +def entrypoint(config, env_name, wraps_module): + # parse config + configs = {} + load_config_from_env(configs, env_name) + load_config_from_file(configs, config) + task_config = PreTaskConfigSpec.parse_obj(configs) + task_config.conf.logger.install() + logger = logging.getLogger(__name__) + logger.debug("logger installed") + logger.debug(f"task config: {task_config}") + FlowHub.load_components_wraps(config=task_config, module_name=wraps_module).run() + + +@component.command() +@click.option("--config", required=False, type=click.File(), help="config path") +@click.option("--env-name", required=False, type=str, help="env name for config") +@click.option("--wraps-module", required=False, type=str, help="component run wraps module") +def cleanup(config, env_name, wraps_module=None): + configs = {} + load_config_from_env(configs, env_name) + load_config_from_file(configs, config) + task_config = PreTaskConfigSpec.parse_obj(configs) + task_config.conf.logger.install() + logger = logging.getLogger(__name__) + logger.debug("logger installed") + logger.debug(f"task config: {task_config}") + FlowHub.load_components_wraps(config=task_config, module_name=wraps_module).cleanup() + + +@component.command() +@click.option("--config", required=False, type=click.File(), help="config path") +@click.option("--env-name", required=False, type=str, help="env name for config") +@click.option( + "--execution-final-meta-path", + type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True), + default=os.path.join(os.getcwd(), "execution_final_meta.json"), + show_default=True, + help="path for execution meta generated by component when execution finished", +) +def execute(config, env_name, execution_final_meta_path): + # parse config + configs = {} + load_config_from_env(configs, env_name) + load_config_from_file(configs, config) + task_config = TaskConfigSpec.parse_obj(configs) + task_config.conf.logger.install() + logger = logging.getLogger(__name__) + logger.debug("logger installed") + logger.debug(f"task config: {task_config}") + os.makedirs(os.path.dirname(execution_final_meta_path), exist_ok=True) + try: + io_meta = execute_component(task_config) + with open(execution_final_meta_path, "w") as fw: + json.dump(dict(status=dict(code=0), io_meta=io_meta.dict()), fw, indent=4) + except Exception as e: + with open(execution_final_meta_path, "w") as fw: + json.dump(dict(status=dict(code=-1, exceptions=traceback.format_exc())), fw) + raise e + + +def load_config_from_file(configs, config_file): + from ruamel import yaml + + if config_file is not None: + configs.update(yaml.safe_load(config_file)) + return configs + + +def load_config_from_env(configs, env_name): + import os + from ruamel import yaml + + if env_name is not None and os.environ.get(env_name): + configs.update(yaml.safe_load(os.environ[env_name])) + return configs + + +def execute_component(config: TaskConfigSpec): + component = load_component(config.component) + logger.info(f"parameters： {config.parameters}") + inputs = IOMeta.InputMeta(data={}, model={}) + outputs = IOMeta.OutputMeta(data={}, model={}, metric={}) + component.execute(config, outputs) + return IOMeta(inputs=inputs, outputs=outputs) + + +def load_component(cpn_name: str): + from fate_flow.components.components import BUILDIN_COMPONENTS + + for cpn in BUILDIN_COMPONENTS: + if cpn.name == cpn_name: + return cpn diff --git a/python/fate_flow/components/model_loader.py b/python/fate_flow/components/model_loader.py deleted file mode 100644 index 57f24904e..000000000 --- a/python/fate_flow/components/model_loader.py +++ /dev/null @@ -1,161 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_flow.components._base import ( - BaseParam, ComponentBase, - ComponentInputProtocol, ComponentMeta, -) -from fate_flow.entity import MetricMeta, MetricType -from fate_flow.model.checkpoint import CheckpointManager -from fate_flow.model.sync_model import SyncComponent -from fate_flow.pipelined_model.pipelined_model import PipelinedModel -from fate_flow.settings import ENABLE_MODEL_STORE -from fate_flow.utils.log_utils import getLogger -from fate_flow.utils.model_utils import gen_party_model_id - - -LOGGER = getLogger() -model_loader_cpn_meta = ComponentMeta('ModelLoader') - - -@model_loader_cpn_meta.bind_runner.on_guest.on_host.on_arbiter -class ModelLoader(ComponentBase): - """ ModelLoader is a component for loading models trained by previous jobs. - - `self.model_id`, `self.model_version`, `self.component_name` and `self.model_alias` - come from the previous job. However, most of the data in `self.tracker` belongs to the current job. - Such as `self.tracker.job_id`, `self.tracker.task_id`, `self.tracker.task_version`, etc. - Be careful when using them. - """ - - def __init__(self): - super().__init__() - self.serialize = False - - self.model_id = None - self.model_version = None - self.component_name = None - self.model_alias = None - self.step_index = None - self.step_name = None - - def read_component_model(self): - pipelined_model = PipelinedModel(gen_party_model_id( - self.model_id, self.tracker.role, self.tracker.party_id - ), self.model_version) - - if self.model_alias is None: - self.model_alias = pipelined_model.get_model_alias(self.component_name) - - component_model = pipelined_model._read_component_model(self.component_name, self.model_alias) - if not component_model: - raise ValueError('The component model is empty.') - - self.model_output = component_model - self.tracker.set_metric_meta('model_loader', f'{self.component_name}-{self.model_alias}', - MetricMeta('component_model', MetricType.COMPONENT_MODEL_INFO, { - 'model_id': self.model_id, - 'model_version': self.model_version, - 'component_name': self.component_name, - 'model_alias': self.model_alias, - })) - - def read_checkpoint(self): - checkpoint_manager = CheckpointManager( - role=self.tracker.role, party_id=self.tracker.party_id, - model_id=self.model_id, model_version=self.model_version, - component_name=self.component_name, - ) - checkpoint_manager.load_checkpoints_from_disk() - - if self.step_index is not None: - checkpoint = checkpoint_manager.get_checkpoint_by_index(self.step_index) - elif self.step_name is not None: - checkpoint = checkpoint_manager.get_checkpoint_by_name(self.step_name) - else: - checkpoint = checkpoint_manager.latest_checkpoint - - if checkpoint is None: - raise ValueError('The checkpoint was not found.') - - data = checkpoint.read(include_database=True) - data['model_id'] = checkpoint_manager.model_id - data['model_version'] = checkpoint_manager.model_version - data['component_name'] = checkpoint_manager.component_name - - self.model_output = data.pop('models') - self.tracker.set_metric_meta('model_loader', f'{checkpoint.step_index}-{checkpoint.step_name}', - MetricMeta('checkpoint', MetricType.CHECKPOINT_INFO, data)) - - def _run(self, cpn_input: ComponentInputProtocol): - need_run = cpn_input.parameters.get('need_run', True) - if not need_run: - return - - for k in ('model_id', 'model_version', 'component_name'): - v = cpn_input.parameters.get(k) - if v is None: - raise KeyError(f'The component ModelLoader needs "{k}"') - setattr(self, k, v) - - for k in ('model_alias', 'step_index', 'step_name'): - v = cpn_input.parameters.get(k) - if v is not None: - setattr(self, k, v) - break - - if ENABLE_MODEL_STORE: - sync_component = SyncComponent( - role=self.tracker.role, party_id=self.tracker.party_id, - model_id=self.model_id, model_version=self.model_version, - component_name=self.component_name, - ) - sync_component.download() - - if self.model_alias is not None: - return self.read_component_model() - - if self.step_index is not None or self.step_name is not None: - return self.read_checkpoint() - - try: - return self.read_component_model() - except Exception: - try: - return self.read_checkpoint() - except Exception: - raise EnvironmentError('Unable to find component model and checkpoint. ' - 'Try specifying "model_alias", "step_index" or "step_name".') - -@model_loader_cpn_meta.bind_param -class ModelLoaderParam(BaseParam): - - def __init__(self, model_id: str = None, model_version: str = None, component_name: str = None, - model_alias: str = None, step_index: int = None, step_name: str = None, need_run: bool = True): - self.model_id = model_id - self.model_version = model_version - self.component_name = component_name - self.model_alias = model_alias - self.step_index = step_index - self.step_name = step_name - self.need_run = need_run - - if self.step_index is not None: - self.step_index = int(self.step_index) - - def check(self): - for i in ('model_id', 'model_version', 'component_name'): - if getattr(self, i) is None: - raise KeyError(f"The parameter '{i}' is required.") diff --git a/python/fate_flow/components/model_operation.py b/python/fate_flow/components/model_operation.py deleted file mode 100644 index 71a4cab58..000000000 --- a/python/fate_flow/components/model_operation.py +++ /dev/null @@ -1,109 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_flow.model import mysql_model_storage, tencent_cos_model_storage -from fate_flow.utils.log_utils import getLogger -from fate_flow.components._base import ( - BaseParam, - ComponentBase, - ComponentMeta, - ComponentInputProtocol, -) -from fate_flow.entity.types import ModelStorage - - -LOGGER = getLogger() - - -ModelStorageClassMap = { - ModelStorage.MYSQL.value: mysql_model_storage.MysqlModelStorage, - ModelStorage.TENCENT_COS.value: tencent_cos_model_storage.TencentCOSModelStorage, -} - - -def get_model_storage(parameters): - model_storage = parameters.get("store_address", {}).get("storage") - if not model_storage: - raise TypeError(f"'store_address' is empty.") - if model_storage not in ModelStorageClassMap: - raise ValueError(f"Model storage '{model_storage}' is not supported.") - return ModelStorageClassMap[model_storage]() - - -model_store_cpn_meta = ComponentMeta("ModelStore") - - -@model_store_cpn_meta.bind_param -class ModelStoreParam(BaseParam): - def __init__( - self, - model_id: str = None, - model_version: str = None, - store_address: dict = None, - force_update: bool = False, - ): - self.model_id = model_id - self.model_version = model_version - self.store_address = store_address - self.force_update = force_update - - def check(self): - return True - - -@model_store_cpn_meta.bind_runner.on_local -class ModelStore(ComponentBase): - def _run(self, cpn_input: ComponentInputProtocol): - parameters = cpn_input.parameters - model_storage = get_model_storage(parameters) - model_storage.store( - parameters["model_id"], parameters["model_version"], - parameters["store_address"], parameters.get("force_update", False), - ) - - -model_restore_cpn_meta = ComponentMeta("ModelRestore") - - -@model_restore_cpn_meta.bind_param -class ModelRestoreParam(BaseParam): - def __init__( - self, - model_id: str = None, - model_version: str = None, - store_address: dict = None, - force_update: bool = False, - hash_: str = None, - ): - self.model_id = model_id - self.model_version = model_version - self.store_address = store_address - self.force_update = force_update - self.hash_ = hash_ - - def check(self): - return True - - -@model_restore_cpn_meta.bind_runner.on_local -class ModelRestore(ComponentBase): - def _run(self, cpn_input: ComponentInputProtocol): - parameters = cpn_input.parameters - model_storage = get_model_storage(parameters) - model_storage.restore( - parameters["model_id"], parameters["model_version"], - parameters["store_address"], parameters.get("force_update", False), - parameters.get("hash_"), - ) diff --git a/python/fate_flow/components/param_extract.py b/python/fate_flow/components/param_extract.py deleted file mode 100644 index 7395e06df..000000000 --- a/python/fate_flow/components/param_extract.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# ============================================================================= -# Param Exact Class -# ============================================================================= -import builtins - -PARAM_MAXDEPTH = 5 - - -class ParamExtract(object): - def __init__(self): - self.builtin_types = dir(builtins) - - def parse_param_from_config( - self, param, config_json, valid_check=False, module=None, cpn=None - ): - if config_json is None or type(config_json).__name__ != "dict": - raise Exception( - "config file is not a valid dict type, please have a check!" - ) - - # default_section = type(param).__name__ - if "ComponentParam" not in config_json: - return param - """ - if default_section not in config_json: - return param - """ - - param = self.recursive_parse_param_from_config( - param, - config_json.get("ComponentParam"), - param_parse_depth=0, - valid_check=valid_check, - name=f"{module}#{cpn}", - ) - - return param - - def recursive_parse_param_from_config( - self, param, config_json, param_parse_depth, valid_check, name - ): - if param_parse_depth > PARAM_MAXDEPTH: - raise ValueError("Param define nesting too deep!!!, can not parse it") - - inst_variables = param.__dict__ - - for variable in inst_variables: - attr = getattr(param, variable) - - if type(attr).__name__ in self.builtin_types or attr is None: - if variable in config_json: - option = config_json[variable] - setattr(param, variable, option) - elif variable in config_json: - sub_params = self.recursive_parse_param_from_config( - attr, - config_json.get(variable), - param_parse_depth + 1, - valid_check, - name, - ) - setattr(param, variable, sub_params) - - if valid_check: - redundant = [] - for var in config_json: - if var not in inst_variables: - redundant.append(var) - - if redundant: - raise ValueError(f"cpn `{name}` has redundant parameters {redundant}") - - return param - - @staticmethod - def change_param_to_dict(obj): - ret_dict = {} - - variable_dict = obj.__dict__ - for variable in variable_dict: - attr = getattr(obj, variable) - if attr and type(attr).__name__ not in dir(builtins): - ret_dict[variable] = ParamExtract.change_param_to_dict(attr) - else: - ret_dict[variable] = attr - - return ret_dict - - @staticmethod - def get_not_builtin_types(obj): - ret_dict = {} - - variable_dict = obj.__dict__ - for variable in variable_dict: - attr = getattr(obj, variable) - if attr and type(attr).__name__ not in dir(builtins): - ret_dict[variable] = ParamExtract.get_not_builtin_types(attr) - - return ret_dict diff --git a/python/fate_flow/components/reader.py b/python/fate_flow/components/reader.py deleted file mode 100644 index 44f57ffad..000000000 --- a/python/fate_flow/components/reader.py +++ /dev/null @@ -1,341 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import numpy as np -from fate_arch import session -from fate_arch.abc import AddressABC, StorageTableABC, StorageTableMetaABC -from fate_arch.common import EngineType, log -from fate_arch.common.data_utils import default_output_fs_path, default_output_info -from fate_arch.computing import ComputingEngine -from fate_arch.session import Session -from fate_arch.storage import StorageEngine, StorageTableMeta, StorageTableOrigin -from fate_flow.components._base import ( - BaseParam, - ComponentBase, - ComponentInputProtocol, - ComponentMeta, -) -from fate_flow.errors import ParameterError -from fate_flow.entity import MetricMeta -from fate_flow.entity.types import InputSearchType -from fate_flow.manager.data_manager import DataTableTracker, TableStorage, AnonymousGenerator -from fate_flow.operation.job_tracker import Tracker -from fate_flow.utils import data_utils -from federatedml.feature.instance import Instance - -LOGGER = log.getLogger() -MAX_NUM = 10000 - -reader_cpn_meta = ComponentMeta("Reader") - - -@reader_cpn_meta.bind_param -class ReaderParam(BaseParam): - def __init__(self, table=None): - self.table = table - - def check(self): - return True - - -@reader_cpn_meta.bind_runner.on_guest.on_host -class Reader(ComponentBase): - def __init__(self): - super(Reader, self).__init__() - self.parameters = None - self.job_parameters = None - - def _run(self, cpn_input: ComponentInputProtocol): - self.parameters = cpn_input.parameters - self.job_parameters = cpn_input.job_parameters - output_storage_address = self.job_parameters.engines_address[EngineType.STORAGE] - # only support one input table - table_key = [key for key in self.parameters.keys()][0] - - input_table_namespace, input_table_name = self.get_input_table_info( - parameters=self.parameters[table_key], - role=self.tracker.role, - party_id=self.tracker.party_id, - ) - ( - output_table_namespace, - output_table_name, - ) = default_output_info( - task_id=self.tracker.task_id, - task_version=self.tracker.task_version, - output_type="data", - ) - ( - input_table_meta, - output_table_address, - output_table_engine, - ) = self.convert_check( - input_name=input_table_name, - input_namespace=input_table_namespace, - output_name=output_table_name, - output_namespace=output_table_namespace, - computing_engine=self.job_parameters.computing_engine, - output_storage_address=output_storage_address, - ) - sess = Session.get_global() - - input_table = sess.get_table( - name=input_table_meta.get_name(), namespace=input_table_meta.get_namespace() - ) - # update real count to meta info - input_table.count() - # Table replication is required - if input_table_meta.get_engine() != output_table_engine: - LOGGER.info( - f"the {input_table_meta.get_engine()} engine input table needs to be converted to {output_table_engine} engine to support computing engine {self.job_parameters.computing_engine}" - ) - else: - LOGGER.info( - f"the {input_table_meta.get_engine()} input table needs to be transform format" - ) - LOGGER.info("reader create storage session2") - output_table_session = sess.storage(storage_engine=output_table_engine) - output_table = output_table_session.create_table( - address=output_table_address, - name=output_table_name, - namespace=output_table_namespace, - partitions=input_table_meta.partitions, - origin=StorageTableOrigin.READER - ) - self.save_table(src_table=input_table, dest_table=output_table) - # update real count to meta info - output_table_meta = StorageTableMeta( - name=output_table.name, namespace=output_table.namespace - ) - # todo: may be set output data, and executor support pass persistent - self.tracker.log_output_data_info( - data_name=cpn_input.flow_feeded_parameters.get("output_data_name")[0] - if cpn_input.flow_feeded_parameters.get("output_data_name") - else table_key, - table_namespace=output_table_meta.get_namespace(), - table_name=output_table_meta.get_name(), - ) - DataTableTracker.create_table_tracker( - output_table_meta.get_name(), - output_table_meta.get_namespace(), - entity_info={ - "have_parent": True, - "parent_table_namespace": input_table_namespace, - "parent_table_name": input_table_name, - "job_id": self.tracker.job_id, - }, - ) - table_info, anonymous_info, attribute_info = self.data_info_display(output_table_meta) - data_info = { - "table_name": input_table_name, - "namespace": input_table_namespace, - "table_info": table_info, - "anonymous_info": anonymous_info, - "attribute_info": attribute_info, - "partitions": output_table_meta.get_partitions(), - "storage_engine": output_table_meta.get_engine(), - } - if input_table_meta.get_engine() in [StorageEngine.PATH]: - data_info["file_count"] = output_table_meta.get_count() - data_info["file_path"] = input_table_meta.get_address().path - else: - data_info["count"] = output_table_meta.get_count() - - self.tracker.set_metric_meta( - metric_namespace="reader_namespace", - metric_name="reader_name", - metric_meta=MetricMeta( - name="reader", metric_type="data_info", extra_metas=data_info - ), - ) - - @staticmethod - def get_input_table_info(parameters, role, party_id): - search_type = data_utils.get_input_search_type(parameters) - if search_type is InputSearchType.TABLE_INFO: - return parameters["namespace"], parameters["name"] - elif search_type is InputSearchType.JOB_COMPONENT_OUTPUT: - output_data_infos = Tracker.query_output_data_infos( - job_id=parameters["job_id"], - component_name=parameters["component_name"], - data_name=parameters["data_name"], - role=role, - party_id=party_id, - ) - if not output_data_infos: - raise Exception(f"can not found input table, please check parameters") - else: - namespace, name = ( - output_data_infos[0].f_table_namespace, - output_data_infos[0].f_table_name, - ) - LOGGER.info(f"found input table {namespace} {name} by {parameters}") - return namespace, name - else: - raise ParameterError( - f"can not found input table info by parameters {parameters}" - ) - - @staticmethod - def convert_check( - input_name, - input_namespace, - output_name, - output_namespace, - computing_engine: ComputingEngine = ComputingEngine.EGGROLL, - output_storage_address={}, - ) -> (StorageTableMetaABC, AddressABC, StorageEngine): - return data_utils.convert_output(input_name, input_namespace, output_name, output_namespace, computing_engine, - output_storage_address) - - def save_table(self, src_table: StorageTableABC, dest_table: StorageTableABC): - LOGGER.info(f"start copying table") - LOGGER.info( - f"source table name: {src_table.name} namespace: {src_table.namespace} engine: {src_table.engine}" - ) - LOGGER.info( - f"destination table name: {dest_table.name} namespace: {dest_table.namespace} engine: {dest_table.engine}" - ) - if src_table.engine == dest_table.engine and src_table.meta.get_in_serialized(): - self.to_save(src_table, dest_table) - else: - TableStorage.copy_table(src_table, dest_table) - # update anonymous - self.create_anonymous(src_meta=src_table.meta, dest_meta=dest_table.meta) - - def to_save(self, src_table, dest_table): - src_table_meta = src_table.meta - src_computing_table = session.get_computing_session().load( - src_table_meta.get_address(), - schema=src_table_meta.get_schema(), - partitions=src_table_meta.get_partitions(), - id_delimiter=src_table_meta.get_id_delimiter(), - in_serialized=src_table_meta.get_in_serialized(), - ) - schema = src_table_meta.get_schema() - self.tracker.job_tracker.save_output_data( - src_computing_table, - output_storage_engine=dest_table.engine, - output_storage_address=dest_table.address.__dict__, - output_table_namespace=dest_table.namespace, - output_table_name=dest_table.name, - schema=schema, - need_read=False - ) - schema = self.update_anonymous(computing_table=src_computing_table,schema=schema, src_table_meta=src_table_meta) - LOGGER.info(f"dest schema: {schema}") - dest_table.meta.update_metas( - schema=schema, - part_of_data=src_table_meta.get_part_of_data(), - count=src_table_meta.get_count(), - id_delimiter=src_table_meta.get_id_delimiter() - ) - LOGGER.info( - f"save {dest_table.namespace} {dest_table.name} success" - ) - return src_computing_table - - def update_anonymous(self, computing_table, schema, src_table_meta): - if schema.get("meta"): - if "anonymous_header" not in schema: - schema.update(AnonymousGenerator.generate_header(computing_table, schema)) - schema = AnonymousGenerator.generate_anonymous_header(schema=schema) - src_table_meta.update_metas(schema=schema) - schema = AnonymousGenerator.update_anonymous_header_with_role(schema, self.tracker.role, self.tracker.party_id) - return schema - - def create_anonymous(self, src_meta, dest_meta): - src_schema = src_meta.get_schema() - dest_schema = dest_meta.get_schema() - LOGGER.info(f"src schema: {src_schema}, dest schema {dest_schema}") - if src_schema.get("meta"): - if "anonymous_header" not in src_schema: - LOGGER.info("start to create anonymous") - dest_computing_table = session.get_computing_session().load( - dest_meta.get_address(), - schema=dest_meta.get_schema(), - partitions=dest_meta.get_partitions(), - id_delimiter=dest_meta.get_id_delimiter(), - in_serialized=dest_meta.get_in_serialized(), - ) - src_schema.update(AnonymousGenerator.generate_header(dest_computing_table, src_schema)) - dest_schema.update(AnonymousGenerator.generate_header(dest_computing_table, dest_schema)) - src_schema = AnonymousGenerator.generate_anonymous_header(schema=src_schema) - dest_schema = AnonymousGenerator.generate_anonymous_header(schema=dest_schema) - dest_schema = AnonymousGenerator.update_anonymous_header_with_role(dest_schema, self.tracker.role, - self.tracker.party_id) - LOGGER.info(f"update src schema {src_schema} and dest schema {dest_schema}") - src_meta.update_metas(schema=src_schema) - dest_meta.update_metas(schema=dest_schema) - else: - dest_schema = AnonymousGenerator.update_anonymous_header_with_role(dest_schema, self.tracker.role, - self.tracker.party_id) - LOGGER.info(f"update dest schema {dest_schema}") - dest_meta.update_metas(schema=dest_schema) - - @staticmethod - def data_info_display(output_table_meta): - headers = output_table_meta.get_schema().get("header") - schema = output_table_meta.get_schema() - table_info = {} - anonymous_info = {} - attribute_info = {} - try: - if schema and headers: - if schema.get("original_index_info"): - data_list = [AnonymousGenerator.reconstruct_header(schema)] - else: - if isinstance(headers, str): - data_list = [headers.split(",")] - else: - data_list = [[schema.get("label_name")] if schema.get("label_name") else []] - data_list[0].extend(headers) - LOGGER.info(f"data info header: {data_list[0]}") - for data in output_table_meta.get_part_of_data(): - if isinstance(data[1], str): - delimiter = schema.get("meta", {}).get( - "delimiter") or output_table_meta.id_delimiter - data_list.append(data[1].split(delimiter)) - elif isinstance(data[1], Instance): - table_data = [] - if data[1].inst_id: - table_data = table_data.append(data[1].inst_id) - if not data[1].label is None: - table_data.append(data[1].label) - - table_data.extend(data[1].features) - data_list.append([str(v) for v in table_data]) - else: - data_list.append(data[1]) - - data = np.array(data_list) - Tdata = data.transpose() - for data in Tdata: - table_info[data[0]] = ",".join(list(set(data[1:]))[:5]) - if schema and schema.get("anonymous_header"): - anonymous_info = dict(zip(schema.get("header"), schema.get("anonymous_header"))) - attribute_info = dict(zip(schema.get("header"), ["feature"] * len(schema.get("header")))) - if schema.get("label_name"): - anonymous_info[schema.get("label_name")] = schema.get("anonymous_label") - attribute_info[schema.get("label_name")] = "label" - if schema.get("meta", {}).get("id_list"): - for id_name in schema.get("meta").get("id_list"): - if id_name in attribute_info: - attribute_info[id_name] = "match_id" - except Exception as e: - LOGGER.exception(e) - return table_info, anonymous_info, attribute_info diff --git a/python/fate_flow/components/upload.py b/python/fate_flow/components/upload.py deleted file mode 100644 index c07377266..000000000 --- a/python/fate_flow/components/upload.py +++ /dev/null @@ -1,423 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os -import shutil -import sys -import time -import uuid -import ast - -from fate_arch import storage, session -from fate_arch.common import EngineType, log, path_utils -from fate_arch.common.data_utils import default_input_fs_path -from fate_arch.session import Session -from fate_arch.storage import DEFAULT_ID_DELIMITER, EggRollStoreType, StorageEngine, StorageTableOrigin -from fate_flow.components._base import ( - BaseParam, - ComponentBase, - ComponentMeta, - ComponentInputProtocol, -) -from fate_flow.components.param_extract import ParamExtract -from fate_flow.entity import Metric, MetricMeta, MetricType -from fate_flow.manager.data_manager import DataTableTracker, AnonymousGenerator, SchemaMetaParam -from fate_flow.scheduling_apps.client import ControllerClient -from fate_flow.db.job_default_config import JobDefaultConfig -from fate_flow.utils import data_utils, job_utils -from fate_flow.utils.base_utils import get_fate_flow_directory - -LOGGER = log.getLogger() - -upload_cpn_meta = ComponentMeta("Upload") - - -@upload_cpn_meta.bind_param -class UploadParam(BaseParam): - def __init__( - self, - file="", - head=1, - id_delimiter=DEFAULT_ID_DELIMITER, - partition=10, - namespace="", - name="", - storage_engine="", - storage_address=None, - destroy=False, - extend_sid=False, - auto_increasing_sid=False, - block_size=1, - schema=None, - # extra param - with_meta=False, - meta={} - ): - self.file = file - self.head = head - self.id_delimiter = id_delimiter - self.partition = partition - self.namespace = namespace - self.name = name - self.storage_engine = storage_engine - self.storage_address = storage_address - self.destroy = destroy - self.extend_sid = extend_sid - self.auto_increasing_sid = auto_increasing_sid - self.block_size = block_size - self.schema = schema if schema else {} - # extra param - self.with_meta = with_meta - self.meta = meta - - def check(self): - return True - - def update(self, conf, allow_redundant=False): - LOGGER.info(f"update:{conf}") - params = ParamExtract().recursive_parse_param_from_config( - param=self, - config_json=conf, - param_parse_depth=0, - valid_check=not allow_redundant, - name=self._name, - ) - params.update_meta(params) - LOGGER.info(f"update result:{params.__dict__}") - return params - - @staticmethod - def update_meta(params): - if params.with_meta: - if isinstance(params.meta, str): - _meta = SchemaMetaParam(**ast.literal_eval(params.meta)).to_dict() - else: - _meta = SchemaMetaParam(**params.meta).to_dict() - if params.extend_sid: - _meta["with_match_id"] = True - else: - _meta = {} - params.meta = _meta - return params - - -@upload_cpn_meta.bind_runner.on_local -class Upload(ComponentBase): - def __init__(self): - super(Upload, self).__init__() - self.MAX_PARTITIONS = 1024 - self.MAX_BYTES = 1024 * 1024 * 8 * 500 - self.parameters = {} - self.table = None - self.is_block = False - self.session_id = None - self.session = None - self.storage_engine = None - - def _run(self, cpn_input: ComponentInputProtocol): - self.parameters = cpn_input.parameters - LOGGER.info(self.parameters) - self.parameters["role"] = cpn_input.roles["role"] - self.parameters["local"] = cpn_input.roles["local"] - storage_engine = self.parameters["storage_engine"].upper() - storage_address = self.parameters["storage_address"] - # if not set storage, use job storage as default - if not storage_engine: - storage_engine = cpn_input.job_parameters.storage_engine - self.storage_engine = storage_engine - if not storage_address: - storage_address = cpn_input.job_parameters.engines_address[ - EngineType.STORAGE - ] - job_id = self.task_version_id.split("_")[0] - if not os.path.isabs(self.parameters.get("file", "")): - self.parameters["file"] = os.path.join( - get_fate_flow_directory(), self.parameters["file"] - ) - if not os.path.exists(self.parameters["file"]): - raise Exception( - "%s is not exist, please check the configure" - % (self.parameters["file"]) - ) - if not os.path.getsize(self.parameters["file"]): - raise Exception("%s is an empty file" % (self.parameters["file"])) - name, namespace = self.parameters.get("name"), self.parameters.get("namespace") - _namespace, _table_name = self.generate_table_name(self.parameters["file"]) - if namespace is None: - namespace = _namespace - if name is None: - name = _table_name - if self.parameters.get("with_meta"): - self.parameters["id_delimiter"] = self.parameters.get("meta").get("delimiter") - read_head = self.parameters["head"] - if read_head == 0: - head = False - elif read_head == 1: - head = True - else: - raise Exception("'head' in conf.json should be 0 or 1") - partitions = self.parameters["partition"] - if partitions <= 0 or partitions >= self.MAX_PARTITIONS: - raise Exception( - "Error number of partition, it should between %d and %d" - % (0, self.MAX_PARTITIONS) - ) - self.session_id = job_utils.generate_session_id( - self.tracker.task_id, - self.tracker.task_version, - self.tracker.role, - self.tracker.party_id, - ) - sess = Session.get_global() - self.session = sess - if self.parameters.get("destroy", False): - table = sess.get_table(namespace=namespace, name=name) - if table: - LOGGER.info( - f"destroy table name: {name} namespace: {namespace} engine: {table.engine}" - ) - try: - table.destroy() - except Exception as e: - LOGGER.error(e) - else: - LOGGER.info( - f"can not found table name: {name} namespace: {namespace}, pass destroy" - ) - address_dict = storage_address.copy() - storage_session = sess.storage( - storage_engine=storage_engine, options=self.parameters.get("options") - ) - upload_address = {} - if storage_engine in {StorageEngine.EGGROLL, StorageEngine.STANDALONE}: - upload_address = { - "name": name, - "namespace": namespace, - "storage_type": EggRollStoreType.ROLLPAIR_LMDB, - } - elif storage_engine in {StorageEngine.MYSQL, StorageEngine.HIVE}: - if not address_dict.get("db") or not address_dict.get("name"): - upload_address = {"db": namespace, "name": name} - elif storage_engine in {StorageEngine.PATH}: - upload_address = {"path": self.parameters["file"]} - elif storage_engine in {StorageEngine.HDFS}: - upload_address = { - "path": default_input_fs_path( - name=name, - namespace=namespace, - prefix=address_dict.get("path_prefix"), - ) - } - elif storage_engine in {StorageEngine.LOCALFS}: - upload_address = { - "path": default_input_fs_path( - name=name, - namespace=namespace, - storage_engine=storage_engine - ) - } - else: - raise RuntimeError(f"can not support this storage engine: {storage_engine}") - address_dict.update(upload_address) - LOGGER.info(f"upload to {storage_engine} storage, address: {address_dict}") - address = storage.StorageTableMeta.create_address( - storage_engine=storage_engine, address_dict=address_dict - ) - self.parameters["partitions"] = partitions - self.parameters["name"] = name - self.table = storage_session.create_table(address=address, origin=StorageTableOrigin.UPLOAD, **self.parameters) - if storage_engine not in [StorageEngine.PATH]: - data_table_count = self.save_data_table(job_id, name, namespace, head) - else: - data_table_count = self.get_data_table_count( - self.parameters["file"], name, namespace - ) - self.table.meta.update_metas(in_serialized=True) - DataTableTracker.create_table_tracker( - table_name=name, - table_namespace=namespace, - entity_info={"job_id": job_id, "have_parent": False}, - ) - LOGGER.info("------------load data finish!-----------------") - # rm tmp file - try: - if "{}/fate_upload_tmp".format(job_id) in self.parameters["file"]: - LOGGER.info("remove tmp upload file") - LOGGER.info(os.path.dirname(self.parameters["file"])) - shutil.rmtree(os.path.dirname(self.parameters["file"])) - except: - LOGGER.info("remove tmp file failed") - LOGGER.info("file: {}".format(self.parameters["file"])) - LOGGER.info("total data_count: {}".format(data_table_count)) - LOGGER.info("table name: {}, table namespace: {}".format(name, namespace)) - - def save_data_table(self, job_id, dst_table_name, dst_table_namespace, head=True): - input_file = self.parameters["file"] - input_feature_count = self.get_count(input_file) - self.upload_file(input_file, head, job_id, input_feature_count) - table_count = self.table.count() - metas_info = { - "count": table_count, - "partitions": self.parameters["partition"], - "extend_sid": self.parameters["extend_sid"] - } - if self.parameters.get("with_meta"): - metas_info.update({"schema": self.generate_anonymous_schema()}) - self.table.meta.update_metas(**metas_info) - self.save_meta( - dst_table_namespace=dst_table_namespace, - dst_table_name=dst_table_name, - table_count=table_count, - ) - return table_count - - @staticmethod - def get_count(input_file): - with open(input_file, "r", encoding="utf-8") as fp: - count = 0 - for _ in fp: - count += 1 - return count - - def kv_generator(self, input_feature_count, fp, job_id, part_of_data): - fate_uuid = uuid.uuid1().hex - get_line = self.get_line() - line_index = 0 - LOGGER.info(input_feature_count) - while True: - lines = fp.readlines(JobDefaultConfig.upload_block_max_bytes) - LOGGER.info(JobDefaultConfig.upload_block_max_bytes) - if lines: - for line in lines: - values = line.rstrip().split(self.parameters["id_delimiter"]) - k, v = get_line( - values=values, - line_index=line_index, - extend_sid=self.parameters["extend_sid"], - auto_increasing_sid=self.parameters["auto_increasing_sid"], - id_delimiter=self.parameters["id_delimiter"], - fate_uuid=fate_uuid, - ) - yield k, v - line_index += 1 - if line_index <= 100: - part_of_data.append((k, v)) - save_progress = line_index / input_feature_count * 100 // 1 - job_info = { - "progress": save_progress, - "job_id": job_id, - "role": self.parameters["local"]["role"], - "party_id": self.parameters["local"]["party_id"], - } - ControllerClient.update_job(job_info=job_info) - else: - return - - def update_schema(self, head, fp): - read_status = False - if head is True: - data_head = fp.readline() - self.update_table_schema(data_head) - read_status = True - else: - self.update_table_schema() - return read_status - - def upload_file(self, input_file, head, job_id=None, input_feature_count=None, table=None): - if not table: - table = self.table - part_of_data = [] - with open(input_file, "r") as fp: - if self.update_schema(head, fp): - input_feature_count -= 1 - self.table.put_all(self.kv_generator(input_feature_count, fp, job_id, part_of_data)) - table.meta.update_metas(part_of_data=part_of_data) - - def get_computing_table(self, name, namespace, schema=None): - storage_table_meta = storage.StorageTableMeta(name=name, namespace=namespace) - computing_table = session.get_computing_session().load( - storage_table_meta.get_address(), - schema=schema if schema else storage_table_meta.get_schema(), - partitions=self.parameters.get("partitions")) - return computing_table - - def generate_anonymous_schema(self): - computing_table = self.get_computing_table(self.table.name, self.table.namespace) - LOGGER.info(f"computing table schema: {computing_table.schema}") - schema = computing_table.schema - if schema.get("meta"): - schema.update(AnonymousGenerator.generate_header(computing_table, schema)) - schema = AnonymousGenerator.generate_anonymous_header(schema=schema) - LOGGER.info(f"extra schema: {schema}") - return schema - - def update_table_schema(self, data_head=""): - LOGGER.info(f"data head: {data_head}") - schema = data_utils.get_header_schema( - header_line=data_head, - id_delimiter=self.parameters["id_delimiter"], - extend_sid=self.parameters["extend_sid"], - ) - # update extra schema and meta info - schema.update(self.parameters.get("schema", {})) - schema.update({"meta": self.parameters.get("meta", {})}) - - _, meta = self.table.meta.update_metas( - schema=schema, - auto_increasing_sid=self.parameters["auto_increasing_sid"], - extend_sid=self.parameters["extend_sid"], - ) - self.table.meta = meta - - def get_line(self): - if not self.parameters["extend_sid"]: - line = data_utils.get_data_line - elif not self.parameters["auto_increasing_sid"]: - line = data_utils.get_sid_data_line - else: - line = data_utils.get_auto_increasing_sid_data_line - return line - - @staticmethod - def generate_table_name(input_file_path): - str_time = time.strftime("%Y%m%d%H%M%S", time.localtime()) - file_name = input_file_path.split(".")[0] - file_name = file_name.split("/")[-1] - return file_name, str_time - - def save_meta(self, dst_table_namespace, dst_table_name, table_count): - self.tracker.log_output_data_info( - data_name="upload", - table_namespace=dst_table_namespace, - table_name=dst_table_name, - ) - self.tracker.log_metric_data( - metric_namespace="upload", - metric_name="data_access", - metrics=[Metric("count", table_count)], - ) - self.tracker.set_metric_meta( - metric_namespace="upload", - metric_name="data_access", - metric_meta=MetricMeta(name="upload", metric_type=MetricType.UPLOAD), - ) - - def get_data_table_count(self, path, name, namespace): - count = path_utils.get_data_table_count(path) - self.save_meta( - dst_table_namespace=namespace, dst_table_name=name, table_count=count - ) - self.table.meta.update_metas(count=count) - return count diff --git a/python/fate_flow/components/writer.py b/python/fate_flow/components/writer.py deleted file mode 100644 index 655ebd686..000000000 --- a/python/fate_flow/components/writer.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_arch.common import log -from fate_arch.common.data_utils import default_output_fs_path -from fate_arch.session import Session -from fate_arch.storage import StorageEngine -from fate_flow.components._base import ( - BaseParam, - ComponentBase, - ComponentInputProtocol, - ComponentMeta, -) -from fate_flow.entity import Metric -from fate_flow.external.data_storage import save_data_to_external_storage -from fate_flow.manager.data_manager import DataTableTracker - -LOGGER = log.getLogger() - -writer_cpn_meta = ComponentMeta("Writer") - - -@writer_cpn_meta.bind_param -class WriterParam(BaseParam): - def __init__(self, - name=None, - namespace=None, - storage_engine=None, - address=None, - output_name=None, - output_namespace=None): - self.name = name - self.namespace = namespace - self.storage_engine = storage_engine - self.address = address - self.output_name = output_name - self.output_namespace = output_namespace - - def check(self): - return True - - -@writer_cpn_meta.bind_runner.on_guest.on_host.on_local -class Writer(ComponentBase): - def __init__(self): - super(Writer, self).__init__() - self.parameters = None - self.job_parameters = None - - def _run(self, cpn_input: ComponentInputProtocol): - self.parameters = cpn_input.parameters - if self.parameters.get("namespace") and self.parameters.get("name"): - namespace = self.parameters.get("namespace") - name = self.parameters.get("name") - elif cpn_input.flow_feeded_parameters.get("table_info"): - namespace = cpn_input.flow_feeded_parameters.get("table_info")[0].get("namespace") - name = cpn_input.flow_feeded_parameters.get("table_info")[0].get("name") - else: - raise Exception("no found name or namespace in input parameters") - LOGGER.info(f"writer parameters:{self.parameters}") - src_table = Session.get_global().get_table(name=name, namespace=namespace) - output_name = self.parameters.get("output_name") - output_namespace = self.parameters.get("output_namespace") - engine = self.parameters.get("storage_engine") - address_dict = self.parameters.get("address") - - if output_name and output_namespace: - table_meta = src_table.meta.to_dict() - address_dict = src_table.meta.get_address().__dict__ - engine = src_table.meta.get_engine() - table_meta.update({ - "name": output_name, - "namespace": output_namespace, - "address": self._create_save_address(engine, address_dict, output_name, output_namespace), - }) - src_table.save_as(**table_meta) - table = src_table.save_as(**table_meta) - table.meta.update_metas(**table_meta) - # output table track - DataTableTracker.create_table_tracker( - name, - namespace, - entity_info={ - "have_parent": True, - "parent_table_namespace": namespace, - "parent_table_name": name, - "job_id": self.tracker.job_id, - } - ) - - elif engine and address_dict: - save_data_to_external_storage(engine, address_dict, src_table) - - LOGGER.info("save success") - self.tracker.log_output_data_info( - data_name="writer", - table_namespace=output_namespace, - table_name=output_name, - ) - self.tracker.log_metric_data( - metric_namespace="writer", - metric_name="writer", - metrics=[Metric("count", src_table.meta.get_count()), - Metric("storage_engine", engine)] - ) - - @staticmethod - def _create_save_address(engine, address_dict, name, namespace): - if engine == StorageEngine.EGGROLL: - address_dict.update({"name": name, "namespace": namespace}) - - elif engine == StorageEngine.STANDALONE: - address_dict.update({"name": name, "namespace": namespace}) - - elif engine == StorageEngine.HIVE: - address_dict.update({"database": namespace, "name": f"{name}"}) - - elif engine == StorageEngine.HDFS: - address_dict.update({"path": default_output_fs_path(name=name, - namespace=namespace, - prefix=address_dict.get("path_prefix"))}) - elif engine == StorageEngine.LOCALFS: - address_dict.update({"path": default_output_fs_path(name=name, namespace=namespace, - storage_engine=StorageEngine.LOCALFS)}) - else: - raise RuntimeError(f"{engine} storage is not supported") - return address_dict diff --git a/python/fate_flow/controller/__init__.py b/python/fate_flow/controller/__init__.py index 1eb6e4b2a..ae946a49c 100644 --- a/python/fate_flow/controller/__init__.py +++ b/python/fate_flow/controller/__init__.py @@ -12,5 +12,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# - diff --git a/python/fate_flow/controller/engine_adapt.py b/python/fate_flow/controller/engine_adapt.py deleted file mode 100644 index b048f1247..000000000 --- a/python/fate_flow/controller/engine_adapt.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_arch.computing import ComputingEngine -from fate_flow.controller.engine_controller.deepspeed import EggrollDeepspeedEngine -from fate_flow.controller.engine_controller.eggroll import EggrollEngine -from fate_flow.controller.engine_controller.linkis_spark import LinkisSparkEngine -from fate_flow.controller.engine_controller.spark import SparkEngine - - -def build_engine(computing_engine, is_deepspeed=False): - if not is_deepspeed: - if computing_engine in {ComputingEngine.EGGROLL, ComputingEngine.STANDALONE}: - engine_session = EggrollEngine() - elif computing_engine == ComputingEngine.SPARK: - engine_session = SparkEngine() - elif computing_engine == ComputingEngine.LINKIS_SPARK: - engine_session = LinkisSparkEngine() - else: - raise ValueError(f"{computing_engine} is not supported") - else: - if computing_engine in {ComputingEngine.EGGROLL, ComputingEngine.STANDALONE}: - engine_session = EggrollDeepspeedEngine() - else: - raise ValueError(f"deepspeed {computing_engine} is not supported") - return engine_session diff --git a/python/fate_flow/controller/engine_controller/deepspeed.py b/python/fate_flow/controller/engine_controller/deepspeed.py deleted file mode 100644 index 7abd0f7c8..000000000 --- a/python/fate_flow/controller/engine_controller/deepspeed.py +++ /dev/null @@ -1,227 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import datetime -import os -import sys -from abc import ABC - -from fate_arch.common.base_utils import json_dumps -from fate_flow.controller.engine_controller.engine import EngineABC -from fate_flow.db.db_models import Task -from fate_flow.db.job_default_config import JobDefaultConfig -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.entity.run_status import BaseStatus, TaskStatus -from fate_flow.entity.types import WorkerName -from fate_flow.manager.worker_manager import WorkerManager -from fate_flow.settings import EXTRA_MODEL_DIR -from fate_flow.utils import log_utils, job_utils, process_utils -from fate_flow.utils.deepspeed_utils import Submit -from fate_flow.utils.log_utils import detect_logger, schedule_logger -from fate_flow.worker.task_executor import TaskExecutor - - -class StatusSet(BaseStatus): - NEW = "NEW" - NEW_TIMEOUT = "NEW_TIMEOUT" - ACTIVE = "ACTIVE" - CLOSED = "CLOSED" - KILLED = "KILLED" - ERROR = "ERROR" - FINISHED = "FINISHED" - - -class EndStatus(BaseStatus): - NEW_TIMEOUT = StatusSet.NEW_TIMEOUT - CLOSED = StatusSet.CLOSED - FAILED = StatusSet.KILLED - ERROR = StatusSet.ERROR - FINISHED = StatusSet.FINISHED - - -class EggrollDeepspeedEngine(EngineABC, ABC): - @staticmethod - def generate_session_id(): - return f"deepspeed_session_{datetime.datetime.now().strftime('%Y%m%d-%H%M%S-%f')}" - - def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_dir, cwd_dir, **kwargs): - worker_id, config_dir, log_dir = WorkerManager.get_process_dirs( - worker_name=WorkerName.TASK_EXECUTOR, - job_id=task.f_job_id, - role=task.f_role, - party_id=task.f_party_id, - task=task - ) - config = run_parameters.to_dict() - session_id, _, command_arguments = WorkerManager.generate_common_cmd(task, config_dir, config, - log_dir, worker_id) - command_arguments.extend(["--is_deepspeed", True]) - command_arguments.extend(["--model_path", self.model_path(task)]) - cmd = [str(_c) for _c in command_arguments] - environment_variables = {} - files = {} - options = { - "eggroll.container.deepspeed.script.path": sys.modules[TaskExecutor.__module__].__file__ - } - task_conf = run_parameters.role_parameter("task_conf", role=task.f_role, party_id=task.f_party_id) - world_size = task_conf.get(task.f_component_name).get("world_size", JobDefaultConfig.task_world_size) - timeout = task_conf.get(task.f_component_name).get("timeout", JobDefaultConfig.resource_waiting_timeout) - resource_options = {"timeout_seconds": timeout, "resource_exhausted_strategy": "waiting"} - submit_conf = { - "world_size": world_size, - "command_arguments": cmd, - "environment_variables": environment_variables, - "files": files, - "resource_options": resource_options, - "options": options - } - config_dir = job_utils.get_task_directory( - task.f_job_id, task.f_role, task.f_party_id, - task.f_component_name, task.f_task_id, str(task.f_task_version) - ) - os.makedirs(config_dir, exist_ok=True) - config_path = os.path.join(config_dir, 'deepspeed_submit.json') - with open(config_path, 'w') as fw: - fw.write(json_dumps(submit_conf)) - session_id = self.generate_session_id() - process_cmd, pid = self.submit(task, config_path, session_id, log_dir=log_dir) - WorkerManager.save_worker_info(task=task, worker_name=WorkerName.TASK_EXECUTOR, worker_id=worker_id, - run_ip=RuntimeConfig.JOB_SERVER_HOST, run_pid=pid, cmd=process_cmd) - return {"worker_id": worker_id, "cmd": cmd, "deepspeed_id": session_id, "run_pid": pid} - - @staticmethod - def submit(task, config_path, session_id, log_dir): - conf_dir = job_utils.get_job_directory(job_id=task.f_job_id) - os.makedirs(conf_dir, exist_ok=True) - process_cmd = [ - sys.executable or 'python3', - sys.modules[Submit.__module__].__file__, - '--job_id', task.f_job_id, - '--role', task.f_role, - '--party_id', task.f_party_id, - '--task_id', task.f_task_id, - '--task_version', task.f_task_version, - '--component_name', task.f_component_name, - '--config', config_path, - '--job_server', f"{RuntimeConfig.JOB_SERVER_HOST}:{RuntimeConfig.HTTP_PORT}", - '--session_id', session_id, - '--log_dir', log_dir, - "--parent_log_dir", os.path.dirname(log_dir) - ] - process_name = "deepspeed_submit" - log_dir = job_utils.get_job_log_directory(job_id=task.f_job_id) - - p = process_utils.run_subprocess(job_id=task.f_job_id, config_dir=conf_dir, process_cmd=process_cmd, - log_dir=log_dir, process_name=process_name) - schedule_logger(task.f_job_id).info(f"run subprocess {p.pid}") - return process_cmd, p.pid - - def kill(self, task): - if task.f_deepspeed_id: - schedule_logger(task.f_job_id).info(f"start kill deepspeed task: {task.f_deepspeed_id}") - from eggroll.deepspeed.submit import client - client = client.DeepspeedJob(task.f_deepspeed_id) - return client.kill() - - @staticmethod - def _query_status(task): - if task.f_deepspeed_id: - from eggroll.deepspeed.submit import client - client = client.DeepspeedJob(task.f_deepspeed_id) - _s = client.query_status().status - return _s if _s else StatusSet.NEW - return StatusSet.NEW - - @staticmethod - def _download_job(task, base_dir, content_type=None, ranks: list = None): - from eggroll.deepspeed.submit import client - if not content_type: - content_type = client.ContentType.ALL - if task.f_deepspeed_id: - client = client.DeepspeedJob(task.f_deepspeed_id) - os.makedirs(base_dir, exist_ok=True) - path = lambda rank: f"{base_dir}/{rank}.zip" - client.download_job_to(rank_to_path=path, content_type=content_type, ranks=ranks) - return base_dir - - def query_task_status(self, task): - status = self._query_status(task) - if status in EndStatus.status_list(): - if status in [EndStatus.FINISHED]: - return TaskStatus.SUCCESS - else: - return TaskStatus.FAILED - - def is_alive(self, task: Task): - status = self._query_status(task) - detect_logger(task.f_job_id).info(f"task {task.f_task_id} {task.f_task_version} deepspeed status {status}") - if status in StatusSet.status_list(): - if status in EndStatus.status_list(): - return False - else: - return True - else: - raise RuntimeError(f"task run status: {status}") - - def download(self, task, base_dir, content_type=None, ranks=None): - from eggroll.deepspeed.submit.client import ContentType - if not content_type: - content_type = ContentType.ALL - dir_name = self._download_job(task, base_dir, content_type, ranks) - if dir_name: - for file in os.listdir(dir_name): - if file.endswith(".zip"): - rank_dir = os.path.join(dir_name, file.split(".zip")[0]) - os.makedirs(rank_dir, exist_ok=True) - self.unzip(os.path.join(dir_name, file), extra_dir=rank_dir) - os.remove(os.path.join(dir_name, file)) - - def download_log(self, task, path=None): - from eggroll.deepspeed.submit.client import ContentType - if not path: - path = self.log_path(task) - self.download(task, base_dir=path, content_type=ContentType.LOGS) - - def download_model(self, task, path=None): - from eggroll.deepspeed.submit.client import ContentType - if not path: - path = self.model_path(task, download=True) - self.download(task, base_dir=path, content_type=ContentType.MODELS, ranks=[0]) - - @staticmethod - def unzip(zip_path, extra_dir): - import zipfile - zfile = zipfile.ZipFile(zip_path, "r") - for name in zfile.namelist(): - dir_name = os.path.dirname(zip_path) - file_path = os.path.join(dir_name, extra_dir, name) - os.makedirs(os.path.dirname(file_path), exist_ok=True) - data = zfile.read(name) - with open(file_path, "w+b") as file: - file.write(data) - - @staticmethod - def model_path(task, download=False): - _p = os.path.join(EXTRA_MODEL_DIR, task.f_job_id, task.f_component_name) - if not download: - # only rank 0 output model - _p = os.path.join(_p, "0") - return _p - - @staticmethod - def log_path(task): - return os.path.join( - log_utils.get_logger_base_dir(), task.f_job_id, task.f_role, task.f_party_id, task.f_component_name - ) diff --git a/python/fate_flow/controller/engine_controller/eggroll.py b/python/fate_flow/controller/engine_controller/eggroll.py deleted file mode 100644 index dcbda78b5..000000000 --- a/python/fate_flow/controller/engine_controller/eggroll.py +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from fate_flow.controller.engine_controller.engine import EngineABC -from fate_flow.db.db_models import Task -from fate_flow.entity.run_status import TaskStatus -from fate_flow.entity.types import KillProcessRetCode, WorkerName -from fate_flow.manager.worker_manager import WorkerManager -from fate_flow.utils import job_utils, process_utils - - -class EggrollEngine(EngineABC): - def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_dir, cwd_dir, **kwargs): - return WorkerManager.start_task_worker(worker_name=WorkerName.TASK_EXECUTOR, task=task, - task_parameters=run_parameters) - - def kill(self, task): - kill_status_code = process_utils.kill_task_executor_process(task) - # session stop - if kill_status_code is KillProcessRetCode.KILLED or task.f_status not in {TaskStatus.WAITING}: - job_utils.start_session_stop(task) - - def is_alive(self, task): - return process_utils.check_process(pid=int(task.f_run_pid), task=task) diff --git a/python/fate_flow/controller/engine_controller/linkis_spark.py b/python/fate_flow/controller/engine_controller/linkis_spark.py deleted file mode 100644 index 21104e1f9..000000000 --- a/python/fate_flow/controller/engine_controller/linkis_spark.py +++ /dev/null @@ -1,119 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from copy import deepcopy - -import requests - -from fate_flow.utils.log_utils import schedule_logger -from fate_flow.controller.engine_controller.engine import EngineABC -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.entity.types import KillProcessRetCode -from fate_flow.entity.run_status import LinkisJobStatus -from fate_flow.settings import LINKIS_EXECUTE_ENTRANCE, LINKIS_SUBMIT_PARAMS, LINKIS_RUNTYPE, \ - LINKIS_LABELS, LINKIS_QUERT_STATUS, LINKIS_KILL_ENTRANCE, detect_logger -from fate_flow.db.service_registry import ServerRegistry -from fate_flow.db.db_models import Task - - -class LinkisSparkEngine(EngineABC): - def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_dir, cwd_dir, **kwargs): - linkis_execute_url = "http://{}:{}{}".format(ServerRegistry.LINKIS_SPARK_CONFIG.get("host"), - ServerRegistry.LINKIS_SPARK_CONFIG.get("port"), - LINKIS_EXECUTE_ENTRANCE) - headers = {"Token-Code": ServerRegistry.LINKIS_SPARK_CONFIG.get("token_code"), - "Token-User": kwargs.get("user_name"), - "Content-Type": "application/json"} - schedule_logger(Task.f_job_id).info(f"headers:{headers}") - python_path = ServerRegistry.LINKIS_SPARK_CONFIG.get("python_path") - execution_code = 'import sys\nsys.path.append("{}")\n' \ - 'from fate_flow.worker.task_executor import TaskExecutor\n' \ - 'task_info = TaskExecutor.run_task(job_id="{}",component_name="{}",' \ - 'task_id="{}",task_version={},role="{}",party_id={},' \ - 'run_ip="{}",config="{}",job_server="{}")\n' \ - 'TaskExecutor.report_task_update_to_driver(task_info=task_info)'. \ - format(python_path, task.f_job_id, task.f_component_name, task.f_task_id, task.f_task_version, task.f_role, task.f_party_id, RuntimeConfig.JOB_SERVER_HOST, - run_parameters_path, '{}:{}'.format(RuntimeConfig.JOB_SERVER_HOST, RuntimeConfig.HTTP_PORT)) - schedule_logger(task.f_job_id).info(f"execution code:{execution_code}") - params = deepcopy(LINKIS_SUBMIT_PARAMS) - schedule_logger(task.f_job_id).info(f"spark run parameters:{run_parameters.spark_run}") - for spark_key, v in run_parameters.spark_run.items(): - if spark_key in ["spark.executor.memory", "spark.driver.memory", "spark.executor.instances", - "wds.linkis.rm.yarnqueue"]: - params["configuration"]["startup"][spark_key] = v - data = { - "method": LINKIS_EXECUTE_ENTRANCE, - "params": params, - "executeApplicationName": "spark", - "executionCode": execution_code, - "runType": LINKIS_RUNTYPE, - "source": {}, - "labels": LINKIS_LABELS - } - schedule_logger(task.f_job_id).info(f'submit linkis spark, data:{data}') - task_info = { - "engine_conf": {} - } - task_info["engine_conf"]["data"] = data - task_info["engine_conf"]["headers"] = headers - res = requests.post(url=linkis_execute_url, headers=headers, json=data) - schedule_logger(task.f_job_id).info(f"start linkis spark task: {res.text}") - if res.status_code == 200: - if res.json().get("status"): - raise Exception(f"submit linkis spark failed: {res.json()}") - task_info["engine_conf"]["execID"] = res.json().get("data").get("execID") - task_info["engine_conf"]["taskID"] = res.json().get("data").get("taskID") - schedule_logger(task.f_job_id).info('submit linkis spark success') - else: - raise Exception(f"submit linkis spark failed: {res.text}") - return task_info - - @staticmethod - def kill(task): - linkis_query_url = "http://{}:{}{}".format(ServerRegistry.LINKIS_SPARK_CONFIG.get("host"), - ServerRegistry.LINKIS_SPARK_CONFIG.get("port"), - LINKIS_QUERT_STATUS.replace("execID", - task.f_engine_conf.get("execID"))) - headers = task.f_engine_conf.get("headers") - response = requests.get(linkis_query_url, headers=headers).json() - schedule_logger(task.f_job_id).info(f"querty task response:{response}") - if response.get("data").get("status") != LinkisJobStatus.SUCCESS: - linkis_execute_url = "http://{}:{}{}".format(ServerRegistry.LINKIS_SPARK_CONFIG.get("host"), - ServerRegistry.LINKIS_SPARK_CONFIG.get("port"), - LINKIS_KILL_ENTRANCE.replace("execID", - task.f_engine_conf.get("execID"))) - schedule_logger(task.f_job_id).info(f"start stop task:{linkis_execute_url}") - schedule_logger(task.f_job_id).info(f"headers: {headers}") - kill_result = requests.get(linkis_execute_url, headers=headers) - schedule_logger(task.f_job_id).info(f"kill result:{kill_result}") - if kill_result.status_code == 200: - pass - return KillProcessRetCode.KILLED - - def is_alive(self, task): - process_exist = True - try: - linkis_query_url = "http://{}:{}{}".format(ServerRegistry.LINKIS_SPARK_CONFIG.get("host"), - ServerRegistry.LINKIS_SPARK_CONFIG.get("port"), - LINKIS_QUERT_STATUS.replace("execID", task.f_engine_conf.get("execID"))) - headers = task.f_engine_conf["headers"] - response = requests.get(linkis_query_url, headers=headers).json() - detect_logger.info(response) - if response.get("data").get("status") == LinkisJobStatus.FAILED: - process_exist = False - except Exception as e: - detect_logger.exception(e) - process_exist = False - return process_exist diff --git a/python/fate_flow/controller/engine_controller/spark.py b/python/fate_flow/controller/engine_controller/spark.py deleted file mode 100644 index ece0f213e..000000000 --- a/python/fate_flow/controller/engine_controller/spark.py +++ /dev/null @@ -1,142 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os - -from fate_arch.common import EngineType -from fate_flow.controller.engine_controller.engine import EngineABC -from fate_flow.db.db_models import Task -from fate_flow.db.dependence_registry import DependenceRegistry -from fate_flow.entity import ComponentProvider -from fate_flow.entity.run_status import TaskStatus -from fate_flow.entity.types import KillProcessRetCode, WorkerName, FateDependenceName, FateDependenceStorageEngine -from fate_flow.manager.resource_manager import ResourceManager -from fate_flow.manager.worker_manager import WorkerManager -from fate_flow.utils import job_utils, process_utils -from fate_flow.db.service_registry import ServerRegistry -from fate_flow.settings import DEPENDENT_DISTRIBUTION -from fate_flow.utils.log_utils import schedule_logger - - -class SparkEngine(EngineABC): - def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_dir, cwd_dir, **kwargs): - spark_home = ServerRegistry.FATE_ON_SPARK.get("spark", {}).get("home") - if not spark_home: - try: - import pyspark - spark_home = pyspark.__path__[0] - except ImportError as e: - raise RuntimeError("can not import pyspark") - except Exception as e: - raise RuntimeError("can not import pyspark") - # else: - # raise ValueError(f"spark home must be configured in conf/service_conf.yaml when run on cluster mode") - - # additional configs - spark_submit_config = run_parameters.spark_run - - deploy_mode = spark_submit_config.get("deploy-mode", "client") - if deploy_mode not in ["client"]: - raise ValueError(f"deploy mode {deploy_mode} not supported") - - spark_submit_cmd = os.path.join(spark_home, "bin/spark-submit") - executable = [spark_submit_cmd, f"--name={task.f_task_id}#{task.f_role}"] - for k, v in spark_submit_config.items(): - if k != "conf": - executable.append(f"--{k}={v}") - if "conf" in spark_submit_config: - for ck, cv in spark_submit_config["conf"].items(): - executable.append(f"--conf") - executable.append(f"{ck}={cv}") - extra_env = {} - extra_env["SPARK_HOME"] = spark_home - if DEPENDENT_DISTRIBUTION: - dependence = Dependence() - dependence.init(provider=ComponentProvider(**task.f_provider_info)) - executor_env_pythonpath, executor_python_env, driver_python_env, archives = dependence.get_task_dependence_info() - schedule_logger(task.f_job_id).info(f"executor_env_python {executor_python_env}，" - f"driver_env_python {driver_python_env}， archives {archives}") - executable.append(f'--archives') - executable.append(archives) - executable.append(f'--conf') - executable.append(f'spark.pyspark.python={executor_python_env}') - executable.append(f'--conf') - executable.append(f'spark.executorEnv.PYTHONPATH={executor_env_pythonpath}') - executable.append(f'--conf') - executable.append(f'spark.pyspark.driver.python={driver_python_env}') - return WorkerManager.start_task_worker(worker_name=WorkerName.TASK_EXECUTOR, task=task, - task_parameters=run_parameters, executable=executable, - extra_env=extra_env) - - def kill(self, task): - kill_status_code = process_utils.kill_task_executor_process(task) - # session stop - if kill_status_code is KillProcessRetCode.KILLED or task.f_status not in {TaskStatus.WAITING}: - job_utils.start_session_stop(task) - - def is_alive(self, task): - return process_utils.check_process(pid=int(task.f_run_pid), task=task) - - -class Dependence: - dependence_config = None - - @classmethod - def init(cls, provider): - cls.set_version_dependence(provider) - - @classmethod - def set_version_dependence(cls, provider, storage_engine=FateDependenceStorageEngine.HDFS.value): - dependence_config = {} - for dependence_type in [FateDependenceName.Fate_Source_Code.value, FateDependenceName.Python_Env.value]: - dependencies_storage_info = DependenceRegistry.get_dependencies_storage_meta(storage_engine=storage_engine, - version=provider.version, - type=dependence_type, - get_or_one=True - ) - dependence_config[dependence_type] = dependencies_storage_info.to_dict() - cls.dependence_config = dependence_config - - @classmethod - def get_task_dependence_info(cls): - return cls.get_executor_env_pythonpath(), cls.get_executor_python_env(), cls.get_driver_python_env(), \ - cls.get_archives() - - @classmethod - def get_executor_env_pythonpath(cls): - return cls.dependence_config.get(FateDependenceName.Fate_Source_Code.value).get("f_dependencies_conf").get( - "executor_env_pythonpath") - - @classmethod - def get_executor_python_env(cls): - return cls.dependence_config.get(FateDependenceName.Python_Env.value).get("f_dependencies_conf").get( - "executor_python") - - @classmethod - def get_driver_python_env(cls): - return cls.dependence_config.get(FateDependenceName.Python_Env.value).get("f_dependencies_conf").get( - "driver_python") - - @classmethod - def get_archives(cls, storage_engine=FateDependenceStorageEngine.HDFS.value): - archives = [] - name_node = ResourceManager.get_engine_registration_info(engine_type=EngineType.STORAGE, - engine_name=storage_engine - ).f_engine_config.get("name_node") - for dependence_type in [FateDependenceName.Fate_Source_Code.value, FateDependenceName.Python_Env.value]: - archives.append( - name_node + cls.dependence_config.get(dependence_type).get("f_dependencies_conf").get("archives") - ) - return ','.join(archives) diff --git a/python/fate_flow/controller/federated.py b/python/fate_flow/controller/federated.py new file mode 100644 index 000000000..6d40f9f11 --- /dev/null +++ b/python/fate_flow/controller/federated.py @@ -0,0 +1,183 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from functools import wraps + +from fate_flow.entity.code import FederatedSchedulingStatusCode, ReturnCode +from fate_flow.manager.operation.job_saver import ScheduleJobSaver +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.utils.log_utils import schedule_logger + + +def get_tasks_by_task_id(task_id, roles=None): + tasks = [{"job_id": task.f_job_id, "role": task.f_role, "party_id": task.f_party_id, + "component": task.f_component, "task_id": task.f_task_id, + "task_version": task.f_task_version} for task in + ScheduleJobSaver.query_task(task_id=task_id, only_latest=True)] + if roles: + _tasks = [] + for party in roles: + _role = party.get("role") + _party_ids = party.get("party_id") + for _party_id in _party_ids: + for task in tasks: + if task.get("role") == _role and task.get("party_id") == _party_id: + _tasks.append(task) + return _tasks + return tasks + + +def schedule_job(func): + @wraps(func) + def _inner(*args, **kwargs): + _return = func(*args, **kwargs) + schedule_logger(kwargs.get("job_id")).info(f"job command '{func.__name__}' return: {_return}") + return _return + return _inner + + +def federated(func): + @wraps(func) + def _inner(*args, **kwargs): + federated_response = func(*args, **kwargs) + schedule_logger(kwargs.get("job_id")).info(f"job command '{func.__name__}' return: {federated_response}") + return return_federated_response(federated_response) + return _inner + + +def federated_task(func): + @wraps(func) + def _inner(*args, **kwargs): + task_id = kwargs.get("task_id") + roles = kwargs.get("roles") + tasks = get_tasks_by_task_id(task_id, roles) + if tasks: + _return = func(tasks=tasks, *args, **kwargs) + schedule_logger(tasks[0]["job_id"]).info(f"task command '{func.__name__}' return: {_return}") + return _return + else: + schedule_logger().exception(f"{func.__name__} no found task by task id {task_id}") + return _inner + + +def return_federated_response(federated_response): + retcode_set = set() + for dest_role in federated_response.keys(): + for party_id in federated_response[dest_role].keys(): + retcode_set.add(federated_response[dest_role][party_id]["code"]) + if len(retcode_set) == 1 and ReturnCode.Base.SUCCESS in retcode_set: + federated_scheduling_status_code = FederatedSchedulingStatusCode.SUCCESS + else: + federated_scheduling_status_code = FederatedSchedulingStatusCode.FAILED + return federated_scheduling_status_code, federated_response + + +class FederatedScheduler: + """ + Send commands to party or scheduler + """ + # Job + @classmethod + @federated + def create_job(cls, job_id, roles, initiator_party_id, job_info): + return RuntimeConfig.SCHEDULE_CLIENT.federated.create_job(job_id, roles, initiator_party_id=initiator_party_id, command_body=job_info) + + @classmethod + @federated + def sync_job_status(cls, job_id, roles, job_info): + return RuntimeConfig.SCHEDULE_CLIENT.federated.sync_job_status(job_id, roles, job_info) + + @classmethod + @federated + def resource_for_job(cls, job_id, roles, operation_type): + return RuntimeConfig.SCHEDULE_CLIENT.federated.resource_for_job(job_id, roles, operation_type) + + @classmethod + @federated + def start_job(cls, job_id, roles): + return RuntimeConfig.SCHEDULE_CLIENT.federated.start_job(job_id, roles) + + @classmethod + @federated + def stop_job(cls, job_id, roles): + return RuntimeConfig.SCHEDULE_CLIENT.federated.stop_job(job_id, roles) + + @classmethod + @federated + def update_job(cls, job_id, roles, command_body=None): + return RuntimeConfig.SCHEDULE_CLIENT.federated.update_job(job_id, roles, command_body) + + # task + @classmethod + @federated_task + @federated + def resource_for_task(cls, task_id, operation_type, tasks=None, roles=None): + return RuntimeConfig.SCHEDULE_CLIENT.federated.resource_for_task(tasks=tasks, operation_type=operation_type) + + @classmethod + @federated_task + @federated + def collect_task(cls, task_id, tasks=None): + return RuntimeConfig.SCHEDULE_CLIENT.federated.collect_task(tasks=tasks) + + @classmethod + @federated_task + def sync_task_status(cls, task_id, tasks, command_body): + return RuntimeConfig.SCHEDULE_CLIENT.federated.sync_task_status(tasks=tasks, command_body=command_body) + + @classmethod + @federated_task + def stop_task(cls, task_id, command_body=None, tasks=None): + return RuntimeConfig.SCHEDULE_CLIENT.federated.stop_task(tasks=tasks, + command_body=command_body) + + @classmethod + @federated_task + @federated + def start_task(cls, task_id, tasks=None): + return RuntimeConfig.SCHEDULE_CLIENT.federated.start_task(tasks=tasks) + + @classmethod + @federated_task + @federated + def rerun_task(cls, task_id, task_version, tasks=None): + return RuntimeConfig.SCHEDULE_CLIENT.federated.rerun_task(tasks=tasks, task_version=task_version) + + # scheduler + @classmethod + @schedule_job + def request_create_job(cls, party_id, initiator_party_id, command_body): + return RuntimeConfig.SCHEDULE_CLIENT.scheduler.create_job(party_id, initiator_party_id, command_body) + + @classmethod + @schedule_job + def request_stop_job(cls, party_id, job_id, stop_status=None): + command_body = {"job_id": job_id} + if stop_status: + command_body.update({ + "stop_status": stop_status + }) + return RuntimeConfig.SCHEDULE_CLIENT.scheduler.stop_job(party_id, command_body) + + @classmethod + @schedule_job + def request_rerun_job(cls, party_id, job_id): + command_body = {"job_id": job_id} + return RuntimeConfig.SCHEDULE_CLIENT.scheduler.rerun_job(party_id, command_body) + + @classmethod + @schedule_job + def report_task_to_scheduler(cls, party_id, command_body): + return RuntimeConfig.SCHEDULE_CLIENT.scheduler.report_task(party_id, command_body) diff --git a/python/fate_flow/controller/job.py b/python/fate_flow/controller/job.py new file mode 100644 index 000000000..e168064fe --- /dev/null +++ b/python/fate_flow/controller/job.py @@ -0,0 +1,568 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import shutil +from copy import deepcopy + +from fate_flow.controller.parser import JobParser +from fate_flow.controller.task import TaskController +from fate_flow.db import Job +from fate_flow.engine.storage import Session +from fate_flow.entity.spec.dag import DAGSchema, InheritConfSpec, JobConfSpec +from fate_flow.entity.types import EndStatus, JobStatus, TaskStatus, EngineType, ComputingEngine +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import NoFoundJob, JobParamsError +from fate_flow.manager.outputs.metric import OutputMetric +from fate_flow.manager.outputs.model import PipelinedModel, ModelMeta +from fate_flow.manager.outputs.data import OutputDataTracking +from fate_flow.manager.service.resource_manager import ResourceManager +from fate_flow.manager.operation.job_saver import JobSaver +from fate_flow.controller.federated import FederatedScheduler +from fate_flow.runtime.job_default_config import JobDefaultConfig +from fate_flow.runtime.system_settings import ENGINES, IGNORE_RESOURCE_ROLES, PARTY_ID, LOCAL_PARTY_ID, COMPUTING_CONF +from fate_flow.utils import job_utils +from fate_flow.utils.base_utils import current_timestamp +from fate_flow.utils.job_utils import get_job_log_directory, save_job_dag +from fate_flow.utils.log_utils import schedule_logger + + +class JobController(object): + @classmethod + def request_create_job(cls, schema: DAGSchema, user_name: str = None, is_local=False): + cls.update_job_default_params(schema, is_local=is_local) + cls.check_job_params(schema) + response = FederatedScheduler.request_create_job( + party_id=schema.dag.conf.scheduler_party_id, + initiator_party_id=schema.dag.conf.initiator_party_id, + command_body=schema.dict() + ) + if user_name and response.get("code") == ReturnCode.Base.SUCCESS: + JobSaver.update_job_user(job_id=response.get("job_id"), user_name=user_name) + if response and isinstance(response, dict) and response.get("code") == ReturnCode.Base.SUCCESS: + save_job_dag(job_id=response.get("job_id"), dag=schema.dict(exclude_unset=True)) + return response + + @classmethod + def request_stop_job(cls, job_id, jobs): + schedule_logger(job_id).info(f"stop job on this party") + status = JobStatus.CANCELED + kill_status, kill_details = JobController.stop_jobs(job_id=job_id, stop_status=status) + schedule_logger(job_id).info(f"stop job on this party status {kill_status}") + schedule_logger(job_id).info(f"request stop job to {status}") + response = FederatedScheduler.request_stop_job( + party_id=jobs[0].f_scheduler_party_id, + job_id=job_id, stop_status=status + ) + schedule_logger(job_id).info(f"stop job response: {response}") + return response + + @classmethod + def request_rerun_job(cls, job): + schedule_logger(job.f_job_id).info(f"request rerun job {job.f_job_id}") + response = FederatedScheduler.request_rerun_job(party_id=job.f_scheduler_party_id, job_id=job.f_job_id) + schedule_logger(job.f_job_id).info(f"rerun job response: {response}") + return response + + @classmethod + def create_job(cls, dag, schema_version, job_id: str, role: str, party_id: str): + # create job and task + schedule_logger(job_id).info(f"start create job {job_id} {role} {party_id}") + dag_schema = DAGSchema(dag=dag, schema_version=schema_version) + job_info = { + "job_id": job_id, + "role": role, + "party_id": party_id, + "dag": dag_schema.dict(), + "progress": 0, + "parties": [party.dict() for party in dag_schema.dag.parties], + "initiator_party_id": dag_schema.dag.conf.initiator_party_id, + "scheduler_party_id": dag_schema.dag.conf.scheduler_party_id, + "status": JobStatus.READY, + "model_id": dag_schema.dag.conf.model_id, + "model_version": dag_schema.dag.conf.model_version + } + party_parameters, task_run, task_cores = cls.adapt_party_parameters(dag_schema, role) + schedule_logger(job_id).info(f"party_job_parameters: {party_parameters}") + schedule_logger(job_id).info(f"role {role} party_id {party_id} task run: {task_run}, task cores {task_cores}") + job_info.update(party_parameters) + JobSaver.create_job(job_info=job_info) + # create task + TaskController.create_tasks(job_id, role, party_id, dag_schema, task_run=task_run, task_cores=task_cores) + + @classmethod + def start_job(cls, job_id, role, party_id, extra_info=None): + schedule_logger(job_id).info(f"try to start job on {role} {party_id}") + job_info = { + "job_id": job_id, + "role": role, + "party_id": party_id, + "status": JobStatus.RUNNING, + "start_time": current_timestamp() + } + if extra_info: + schedule_logger(job_id).info(f"extra info: {extra_info}") + job_info.update(extra_info) + try: + cls.inheritance_job(job_id, role, party_id) + except Exception as e: + schedule_logger(job_id).exception(e) + job_info.update({"status": JobStatus.FAILED}) + finally: + cls.update_job_status(job_info=job_info) + cls.update_job(job_info=job_info) + schedule_logger(job_id).info(f"start job on {role} {party_id} {job_info.get('status')}") + + @classmethod + def inheritance_job(cls, job_id, role, party_id): + job = JobSaver.query_job(job_id=job_id, role=role, party_id=party_id)[0] + if job.f_inheritance: + schedule_logger(job_id).info(f"start inherit job {job_id}, inheritance: {job.f_inheritance}") + JobInheritance.load(job) + + @classmethod + def update_job_status(cls, job_info): + update_status = JobSaver.update_job_status(job_info=job_info) + if update_status and EndStatus.contains(job_info.get("status")): + ResourceManager.return_job_resource( + job_id=job_info["job_id"], role=job_info["role"], party_id=job_info["party_id"]) + return update_status + + @classmethod + def stop_jobs(cls, job_id, role=None, party_id=None, stop_status=JobStatus.FAILED): + if not stop_status: + stop_status = JobStatus.FAILED + if role and party_id: + jobs = JobSaver.query_job( + job_id=job_id, role=role, party_id=party_id) + else: + jobs = JobSaver.query_job(job_id=job_id) + kill_status = True + kill_details = {} + for job in jobs: + kill_job_status, kill_job_details = cls.stop_job( + job=job, stop_status=stop_status) + kill_status = kill_status & kill_job_status + kill_details[job_id] = kill_job_details + return kill_status, kill_details + + @classmethod + def stop_job(cls, job, stop_status): + tasks = JobSaver.query_task( + job_id=job.f_job_id, role=job.f_role, party_id=job.f_party_id, only_latest=True, reverse=True) + kill_status = True + kill_details = {} + for task in tasks: + if task.f_status in [TaskStatus.SUCCESS, TaskStatus.WAITING, TaskStatus.PASS]: + continue + kill_task_status = TaskController.stop_task(task, stop_status) + kill_status = kill_status & kill_task_status + kill_details[task.f_task_id] = 'success' if kill_task_status else 'failed' + if kill_status: + job_info = job.to_human_model_dict(only_primary_with=["status"]) + job_info["status"] = stop_status + JobController.update_job_status(job_info) + return kill_status, kill_details + + @classmethod + def update_job(cls, job_info): + return JobSaver.update_job(job_info=job_info) + + @classmethod + def query_job(cls, **kwargs): + query_filters = {} + for k, v in kwargs.items(): + if v is not None: + query_filters[k] = v + return JobSaver.query_job(**query_filters) + + @classmethod + def query_job_list(cls, limit, page, job_id, description, partner, party_id, role, status, order_by, order, + user_name): + # Provided to the job display page + offset = limit * (page - 1) + query = {'tag': ('!=', 'submit_failed')} + if job_id: + query["job_id"] = ('contains', job_id) + if description: + query["description"] = ('contains', description) + if party_id: + query["party_id"] = ('contains', party_id) + # if partner: + # query["parties"] = ('contains', partner) + if role: + query["role"] = ('in_', set(role)) + if status: + query["status"] = ('in_', set(status)) + by = [] + if order_by: + by.append(order_by) + if order: + by.append(order) + if not by: + by = ['create_time', 'desc'] + if user_name: + query["user_name"] = ("==", user_name) + jobs, count = JobSaver.list_job(limit, offset, query, by) + jobs = [job.to_human_model_dict() for job in jobs] + lst_job = [] + for job in jobs: + job['partners'] = set() + for _r in job['parties']: + job['partners'].update(_r.get("party_id")) + job['partners'].discard(job['party_id']) + job['partners'] = sorted(job['partners']) + if partner and str(partner) not in job['partners']: + continue + lst_job.append(job) + return count, lst_job + + @classmethod + def query_task_list(cls, limit, page, job_id, role, party_id, task_name, order_by, order): + offset = limit * (page - 1) + + query = {} + if job_id: + query["job_id"] = job_id + if role: + query["role"] = role + if party_id: + query["party_id"] = party_id + if task_name: + query["task_name"] = task_name + by = [] + if by: + by.append(order_by) + if order: + by.append(order) + if not by: + by = ['create_time', 'desc'] + + tasks, count = JobSaver.list_task(limit, offset, query, by) + return count, [task.to_human_model_dict() for task in tasks] + + @classmethod + def query_tasks(cls, **kwargs): + query_filters = {} + for k, v in kwargs.items(): + if v is not None: + query_filters[k] = v + return JobSaver.query_task(**query_filters) + + @classmethod + def clean_queue(cls): + # stop waiting job + jobs = JobSaver.query_job(status=JobStatus.WAITING) + clean_status = {} + for job in jobs: + status = FederatedScheduler.request_stop_job(party_id=job.f_scheduler_party_id,job_id=job.f_job_id, stop_status=JobStatus.CANCELED) + clean_status[job.f_job_id] = status + return clean_status + + @classmethod + def clean_job(cls, job_id): + jobs = JobSaver.query_job(job_id=job_id) + tasks = JobSaver.query_task(job_id=job_id) + if not jobs: + raise NoFoundJob(job_id=job_id) + FederatedScheduler.request_stop_job( + party_id=jobs[0].f_scheduler_party_id,job_id=jobs[0].f_job_id, stop_status=JobStatus.CANCELED + ) + for task in tasks: + # metric + try: + OutputMetric( + job_id=task.f_job_id, role=task.f_role, party_id=task.f_party_id, task_name=task.f_task_name, + task_id=task.f_task_id, task_version=task.f_task_version + ).delete_metrics() + schedule_logger(task.f_job_id).info(f'delete {task.f_job_id} {task.f_role} {task.f_party_id}' + f' {task.f_task_name} metric data success') + except Exception as e: + schedule_logger(job_id).exception(e) + + # data + try: + datas = OutputDataTracking.query( + job_id=task.f_job_id, + role=task.f_role, + party_id=task.f_party_id, + task_name=task.f_task_name, + task_id=task.f_task_id, + task_version=task.f_task_version + ) + with Session() as sess: + for data in datas: + table = sess.get_table(name=data.f_name, namespace=data.f_namespace) + if table: + table.destroy() + except Exception as e: + schedule_logger(job_id).exception(e) + + # model + try: + PipelinedModel.delete_model(job_id=task.f_job_id, role=task.f_role, + party_id=task.f_party_id, task_name=task.f_task_name) + schedule_logger(task.f_job_id).info(f'delete {task.f_job_id} {task.f_role} {task.f_party_id}' + f' {task.f_task_name} model success') + except Exception as e: + schedule_logger(job_id).exception(e) + + try: + JobSaver.delete_job(job_id=job_id) + except Exception as e: + schedule_logger(job_id).exception(e) + + try: + JobSaver.delete_task(job_id=job_id) + except Exception as e: + schedule_logger(job_id).exception(e) + + @staticmethod + def add_notes(job_id, role, party_id, notes): + job_info = { + "job_id": job_id, + "role": role, + "party_id": party_id, + "description": notes + } + return JobSaver.update_job(job_info) + + @classmethod + def data_view(cls, job_id, role, party_id): + jobs = JobSaver.query_job(job_id=job_id, role=role, party_id=party_id) + if not jobs: + raise NoFoundJob(job_id=job_id, role=role, party_id=party_id) + job = jobs[0] + dag = DAGSchema(**job.f_dag) + job_parser = JobParser(dag) + data_view = {} + for party in dag.dag.parties: + data_view[party.role] = {} + for party_id in party.party_id: + dataset = job_parser.dataset_list(party.role, party_id) + data_view[party.role][party_id] = [data.to_dict() for data in dataset] + return dict(role=role, party_id=party_id, data_view=data_view) + + @classmethod + def adapt_party_parameters(cls, dag_schema: DAGSchema, role): + cores, task_run, task_cores = cls.calculate_resource(dag_schema, role) + job_info = {"cores": cores, "remaining_cores": cores} + if dag_schema.dag.conf.inheritance: + job_info.update({"inheritance": dag_schema.dag.conf.inheritance.dict()}) + return job_info, task_run, task_cores + + @classmethod + def calculate_resource(cls, dag_schema: DAGSchema, role): + cores = dag_schema.dag.conf.cores if dag_schema.dag.conf.cores else JobDefaultConfig.job_cores + if dag_schema.dag.conf.task and dag_schema.dag.conf.task.engine_run: + task_run = dag_schema.dag.conf.task.engine_run + else: + task_run = {} + + task_cores = cores + + default_task_run = deepcopy(JobDefaultConfig.task_run.get(ENGINES.get(EngineType.COMPUTING), {})) + + if ENGINES.get(EngineType.COMPUTING) == ComputingEngine.SPARK: + if "num-executors" not in task_run: + task_run["num-executors"] = default_task_run.get("num-executors") + if "executor-cores" not in task_run: + task_run["executor-cores"] = default_task_run.get("executor-cores") + if role in IGNORE_RESOURCE_ROLES: + task_run["num-executors"] = 1 + task_run["executor-cores"] = 1 + task_cores = int(task_run.get("num-executors")) * (task_run.get("executor-cores")) + + if ENGINES.get(EngineType.COMPUTING) == ComputingEngine.EGGROLL: + total_cores = task_run.get("cores", None) or default_task_run.get("cores") + + task_run["nodes"] = COMPUTING_CONF.get(ComputingEngine.EGGROLL).get("nodes") + task_run["task_cores_per_node"] = max(total_cores // task_run["nodes"], 1) + task_cores = task_run["task_cores_per_node"] * task_run["nodes"] + + if role in IGNORE_RESOURCE_ROLES: + task_run["task_cores_per_node"] = 1 + + if ENGINES.get(EngineType.COMPUTING) == ComputingEngine.STANDALONE: + task_cores = task_run["cores"] = task_run.pop("cores", None) or default_task_run.get("cores") + if role in IGNORE_RESOURCE_ROLES: + task_run["cores"] = 1 + if role in IGNORE_RESOURCE_ROLES: + cores = 0 + task_cores = 0 + + if task_cores > cores: + cores = task_cores + return cores, task_run, task_cores + + @classmethod + def check_job_params(cls, dag_schema: DAGSchema): + # check inheritance + job_utils.inheritance_check(dag_schema.dag.conf.inheritance) + + # check model warehouse + model_warehouse = dag_schema.dag.conf.model_warehouse + if model_warehouse: + if not ModelMeta.query(model_id=model_warehouse.model_id, model_version=model_warehouse.model_version): + raise JobParamsError( + model_id=model_warehouse.model_id, + model_version=model_warehouse.model_version, + position="dag_schema.dag.conf.model_warehouse" + ) + + @classmethod + def update_job_default_params(cls, dag_schema: DAGSchema, is_local: bool = False): + if not dag_schema.dag.conf: + dag_schema.dag.conf = JobConfSpec() + dag_schema.dag.conf.initiator_party_id = PARTY_ID + if not dag_schema.dag.conf.scheduler_party_id: + if not is_local: + dag_schema.dag.conf.scheduler_party_id = PARTY_ID + else: + dag_schema.dag.conf.scheduler_party_id = LOCAL_PARTY_ID + if not dag_schema.dag.conf.computing_partitions: + dag_schema.dag.conf.computing_partitions = JobDefaultConfig.computing_partitions + return dag_schema + + +class JobInheritance: + @classmethod + def load(cls, job: Job): + # load inheritance: data、model、metric、logs + inheritance = InheritConfSpec(**job.f_inheritance) + source_task_list = JobSaver.query_task(job_id=inheritance.job_id, role=job.f_role, party_id=job.f_party_id) + task_list = JobSaver.query_task(job_id=job.f_job_id, role=job.f_role, party_id=job.f_party_id) + target_task_list = [task for task in task_list if task.f_task_name in inheritance.task_list] + cls.load_logs(job, inheritance) + cls.load_output_tracking(job.f_job_id, source_task_list, target_task_list) + cls.load_model_meta(job.f_job_id, source_task_list, target_task_list, job.f_model_id, job.f_model_version) + cls.load_metric(job.f_job_id, source_task_list, target_task_list) + cls.load_status(job.f_job_id, source_task_list, target_task_list) + + @classmethod + def load_logs(cls, job: Job, inheritance: InheritConfSpec): + schedule_logger(job.f_job_id).info("start load job logs") + for task_name in inheritance.task_list: + source_path = os.path.join(get_job_log_directory(inheritance.job_id), job.f_role, job.f_party_id, task_name) + target_path = os.path.join(get_job_log_directory(job.f_job_id), job.f_role, job.f_party_id, task_name) + if os.path.exists(source_path): + if os.path.exists(target_path): + shutil.rmtree(target_path) + shutil.copytree(source_path, target_path) + schedule_logger(job.f_job_id).info("load job logs success") + + @classmethod + def load_output_tracking(cls, job_id, source_task_list, target_task_list): + def callback(target_task, source_task): + output_tracking = OutputDataTracking.query( + job_id=source_task.f_job_id, + role=source_task.f_role, + party_id=source_task.f_party_id, + task_name=source_task.f_task_name, + task_id=source_task.f_task_id, + task_version=source_task.f_task_version + ) + for t in output_tracking: + _t = t.to_human_model_dict() + _t.update({ + "job_id": target_task.f_job_id, + "task_id": target_task.f_task_id, + "task_version": target_task.f_task_version, + "role": target_task.f_role, + "party_id": target_task.f_party_id + }) + OutputDataTracking.create(_t) + schedule_logger(job_id).info("start load output tracking") + cls.load_do(source_task_list, target_task_list, callback) + schedule_logger(job_id).info("load output tracking success") + + @classmethod + def load_model_meta(cls, job_id, source_task_list, target_task_list, model_id, model_version): + def callback(target_task, source_task): + _model_metas = ModelMeta.query( + job_id=source_task.f_job_id, + role=source_task.f_role, + party_id=source_task.f_party_id, + task_name=source_task.f_task_name + ) + for _meta in _model_metas: + _md = _meta.to_human_model_dict() + _md.update({ + "job_id": target_task.f_job_id, + "task_id": target_task.f_task_id, + "task_version": target_task.f_task_version, + "role": target_task.f_role, + "party_id": target_task.f_party_id, + "model_id": model_id, + "model_version": model_version + }) + ModelMeta.save(**_md) + schedule_logger(job_id).info("start load model meta") + cls.load_do(source_task_list, target_task_list, callback) + schedule_logger(job_id).info("load model meta success") + + @classmethod + def load_metric(cls, job_id, source_task_list, target_task_list): + def callback(target_task, source_task): + OutputMetric( + job_id=source_task.f_job_id, + role=source_task.f_role, + party_id=source_task.f_party_id, + task_name=source_task.f_task_name, + task_id=source_task.f_task_id, + task_version=source_task.f_task_version + ).save_as( + job_id=target_task.f_job_id, + role=target_task.f_role, + party_id=target_task.f_party_id, + task_name=target_task.f_task_name, + task_id=target_task.f_task_id, + task_version=target_task.f_task_version + ) + schedule_logger(job_id).info("start load metric") + cls.load_do(source_task_list, target_task_list, callback) + schedule_logger(job_id).info("load metric success") + + @classmethod + def load_status(cls, job_id, source_task_list, target_task_list): + def callback(target_task, source_task): + task_info = { + "job_id": target_task.f_job_id, + "task_id": target_task.f_task_id, + "task_version": target_task.f_task_version, + "role": target_task.f_role, + "party_id": target_task.f_party_id + } + update_info = {} + update_list = ["cmd", "elapsed", "end_time", "engine_conf", "party_status", "run_ip", + "run_pid", "start_time", "status", "worker_id"] + for k in update_list: + update_info[k] = getattr(source_task, f"f_{k}") + task_info.update(update_info) + schedule_logger(task_info["job_id"]).info( + "try to update task {} {}".format(task_info["task_id"], task_info["task_version"])) + schedule_logger(task_info["job_id"]).info("update info: {}".format(update_info)) + JobSaver.update_task(task_info) + TaskController.update_task_status(task_info) + schedule_logger(job_id).info("start load status") + cls.load_do(source_task_list, target_task_list, callback) + schedule_logger(job_id).info("load status success") + + @staticmethod + def load_do(source_task_list, target_task_list, callback): + for source_task in source_task_list: + for target_task in target_task_list: + if target_task.f_task_name == source_task.f_task_name: + callback(target_task, source_task) diff --git a/python/fate_flow/controller/job_controller.py b/python/fate_flow/controller/job_controller.py deleted file mode 100644 index 630f7f182..000000000 --- a/python/fate_flow/controller/job_controller.py +++ /dev/null @@ -1,691 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os -import shutil - -from fate_arch.common import EngineType, engine_utils -from fate_arch.common.base_utils import current_timestamp, json_dumps -from fate_arch.computing import ComputingEngine - -from fate_flow.controller.task_controller import TaskController -from fate_flow.db.db_models import PipelineComponentMeta -from fate_flow.db.job_default_config import JobDefaultConfig -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.entity import RunParameters -from fate_flow.entity.run_status import EndStatus, JobInheritanceStatus, JobStatus, TaskStatus -from fate_flow.entity.types import RetCode, WorkerName -from fate_flow.manager.provider_manager import ProviderManager -from fate_flow.manager.resource_manager import ResourceManager -from fate_flow.manager.worker_manager import WorkerManager -from fate_flow.model.checkpoint import CheckpointManager -from fate_flow.model.sync_model import SyncComponent -from fate_flow.operation.job_saver import JobSaver -from fate_flow.operation.job_tracker import Tracker -from fate_flow.pipelined_model.pipelined_model import PipelinedComponent -from fate_flow.protobuf.python import pipeline_pb2 -from fate_flow.scheduler.federated_scheduler import FederatedScheduler -from fate_flow.settings import ENABLE_MODEL_STORE, ENGINES -from fate_flow.utils import data_utils, job_utils, log_utils, schedule_utils -from fate_flow.utils.job_utils import get_job_dataset -from fate_flow.utils.log_utils import schedule_logger -from fate_flow.utils.model_utils import gather_model_info_data, save_model_info - - -class JobController(object): - @classmethod - def create_job(cls, job_id, role, party_id, job_info): - # parse job configuration - dsl = job_info['dsl'] - runtime_conf = job_info['runtime_conf'] - train_runtime_conf = job_info['train_runtime_conf'] - - dsl_parser = schedule_utils.get_job_dsl_parser( - dsl=dsl, - runtime_conf=runtime_conf, - train_runtime_conf=train_runtime_conf - ) - job_parameters = dsl_parser.get_job_parameters(runtime_conf, int(runtime_conf.get("dsl_version", "1"))) - schedule_logger(job_id).info('job parameters:{}'.format(job_parameters)) - dest_user = job_parameters.get(role, {}).get(party_id, {}).get('user', '') - user = {} - src_party_id = int(job_info['src_party_id']) if job_info.get('src_party_id') else 0 - src_role = job_info.get('src_role', '') - src_user = job_parameters.get(src_role, {}).get(src_party_id, {}).get('user', '') if src_role else '' - for _role, party_id_item in job_parameters.items(): - user[_role] = {} - for _party_id, _parameters in party_id_item.items(): - user[_role][_party_id] = _parameters.get("user", "") - job_parameters = RunParameters(**job_parameters.get(role, {}).get(party_id, {})) - - # save new job into db - if role == job_info["initiator_role"] and party_id == job_info["initiator_party_id"]: - is_initiator = True - else: - is_initiator = False - job_info["status"] = JobStatus.READY - job_info["user_id"] = dest_user - job_info["src_user"] = src_user - job_info["user"] = user - # this party configuration - job_info["role"] = role - job_info["party_id"] = party_id - job_info["is_initiator"] = is_initiator - job_info["progress"] = 0 - cls.create_job_parameters_on_party(role=role, party_id=party_id, job_parameters=job_parameters) - # update job parameters on party - job_info["runtime_conf_on_party"]["job_parameters"] = job_parameters.to_dict() - JobSaver.create_job(job_info=job_info) - schedule_logger(job_id).info("start initialize tasks") - initialized_result, provider_group = cls.initialize_tasks(job_id=job_id, - role=role, - party_id=party_id, - run_on_this_party=True, - initiator_role=job_info["initiator_role"], - initiator_party_id=job_info["initiator_party_id"], - job_parameters=job_parameters, - dsl_parser=dsl_parser, - runtime_conf=runtime_conf, - check_version=True) - schedule_logger(job_id).info("initialize tasks success") - for provider_key, group_info in provider_group.items(): - for cpn in group_info["components"]: - dsl["components"][cpn]["provider"] = provider_key - - roles = job_info['roles'] - cls.initialize_job_tracker(job_id=job_id, role=role, party_id=party_id, - job_parameters=job_parameters, roles=roles, is_initiator=is_initiator, dsl_parser=dsl_parser) - - job_utils.save_job_conf(job_id=job_id, - role=role, - party_id=party_id, - dsl=dsl, - runtime_conf=runtime_conf, - runtime_conf_on_party=job_info["runtime_conf_on_party"], - train_runtime_conf=train_runtime_conf, - pipeline_dsl=None) - return {"components": initialized_result} - - @classmethod - def set_federated_mode(cls, job_parameters: RunParameters): - if not job_parameters.federated_mode: - job_parameters.federated_mode = ENGINES["federated_mode"] - - @classmethod - def set_engines(cls, job_parameters: RunParameters, engine_type=None): - engines = engine_utils.get_engines() - if not engine_type: - engine_type = {EngineType.COMPUTING, EngineType.FEDERATION, EngineType.STORAGE} - for k in engine_type: - setattr(job_parameters, f"{k}_engine", engines[k]) - - @classmethod - def create_common_job_parameters(cls, job_id, initiator_role, common_job_parameters: RunParameters): - JobController.set_federated_mode(job_parameters=common_job_parameters) - JobController.set_engines(job_parameters=common_job_parameters, engine_type={EngineType.COMPUTING}) - JobController.fill_default_job_parameters(job_id=job_id, job_parameters=common_job_parameters) - JobController.adapt_job_parameters(role=initiator_role, job_parameters=common_job_parameters, create_initiator_baseline=True) - - @classmethod - def create_job_parameters_on_party(cls, role, party_id, job_parameters: RunParameters): - JobController.set_engines(job_parameters=job_parameters) - cls.fill_party_specific_parameters(role=role, - party_id=party_id, - job_parameters=job_parameters) - - @classmethod - def fill_party_specific_parameters(cls, role, party_id, job_parameters: RunParameters): - cls.adapt_job_parameters(role=role, job_parameters=job_parameters) - engines_info = cls.get_job_engines_address(job_parameters=job_parameters) - cls.check_parameters(job_parameters=job_parameters, - role=role, party_id=party_id, engines_info=engines_info) - - @classmethod - def fill_default_job_parameters(cls, job_id, job_parameters: RunParameters): - keys = {"task_parallelism", "auto_retries", "auto_retry_delay", "federated_status_collect_type"} - for key in keys: - if hasattr(job_parameters, key) and getattr(job_parameters, key) is None: - if hasattr(JobDefaultConfig, key): - setattr(job_parameters, key, getattr(JobDefaultConfig, key)) - else: - schedule_logger(job_id).warning(f"can not found {key} job parameter default value from job_default_settings") - - @classmethod - def adapt_job_parameters(cls, role, job_parameters: RunParameters, create_initiator_baseline=False): - ResourceManager.adapt_engine_parameters( - role=role, job_parameters=job_parameters, create_initiator_baseline=create_initiator_baseline) - if create_initiator_baseline: - if job_parameters.task_parallelism is None: - job_parameters.task_parallelism = JobDefaultConfig.task_parallelism - if job_parameters.federated_status_collect_type is None: - job_parameters.federated_status_collect_type = JobDefaultConfig.federated_status_collect_type - if create_initiator_baseline and not job_parameters.computing_partitions: - job_parameters.computing_partitions = job_parameters.adaptation_parameters[ - "task_cores_per_node"] * job_parameters.adaptation_parameters["task_nodes"] - - @classmethod - def get_job_engines_address(cls, job_parameters: RunParameters): - engines_info = {} - engine_list = [ - (EngineType.COMPUTING, job_parameters.computing_engine), - (EngineType.FEDERATION, job_parameters.federation_engine), - (EngineType.STORAGE, job_parameters.storage_engine) - ] - for engine_type, engine_name in engine_list: - engine_info = ResourceManager.get_engine_registration_info( - engine_type=engine_type, engine_name=engine_name) - job_parameters.engines_address[engine_type] = engine_info.f_engine_config if engine_info else {} - engines_info[engine_type] = engine_info - return engines_info - - @classmethod - def check_parameters(cls, job_parameters: RunParameters, role, party_id, engines_info): - status, cores_submit, max_cores_per_job = ResourceManager.check_resource_apply( - job_parameters=job_parameters, role=role, party_id=party_id, engines_info=engines_info) - if not status: - msg = "" - msg2 = "default value is fate_flow/settings.py#DEFAULT_TASK_CORES_PER_NODE, refer fate_flow/examples/simple/simple_job_conf.json" - if job_parameters.computing_engine in {ComputingEngine.EGGROLL, ComputingEngine.STANDALONE}: - msg = "please use task_cores job parameters to set request task cores or you can customize it with eggroll_run job parameters" - elif job_parameters.computing_engine in {ComputingEngine.SPARK}: - msg = "please use task_cores job parameters to set request task cores or you can customize it with spark_run job parameters" - raise RuntimeError( - f"max cores per job is {max_cores_per_job} base on (fate_flow/settings#MAX_CORES_PERCENT_PER_JOB * conf/service_conf.yaml#nodes * conf/service_conf.yaml#cores_per_node), expect {cores_submit} cores, {msg}, {msg2}") - - @classmethod - def gen_updated_parameters(cls, job_id, initiator_role, initiator_party_id, input_job_parameters, input_component_parameters): - # todo: check can not update job parameters - job_configuration = job_utils.get_job_configuration(job_id=job_id, - role=initiator_role, - party_id=initiator_party_id) - updated_job_parameters = job_configuration.runtime_conf["job_parameters"] - updated_component_parameters = job_configuration.runtime_conf["component_parameters"] - if input_job_parameters: - if input_job_parameters.get("common"): - common_job_parameters = RunParameters(**input_job_parameters["common"]) - cls.create_common_job_parameters(job_id=job_id, initiator_role=initiator_role, common_job_parameters=common_job_parameters) - for attr in {"model_id", "model_version"}: - setattr(common_job_parameters, attr, updated_job_parameters["common"].get(attr)) - updated_job_parameters["common"] = common_job_parameters.to_dict() - # not support role - updated_components = set() - if input_component_parameters: - cls.merge_update(input_component_parameters, updated_component_parameters) - return updated_job_parameters, updated_component_parameters, list(updated_components) - - @classmethod - def merge_update(cls, inputs: dict, results: dict): - if not isinstance(inputs, dict) or not isinstance(results, dict): - raise ValueError(f"must both dict, but {type(inputs)} inputs and {type(results)} results") - for k, v in inputs.items(): - if k not in results: - results[k] = v - elif isinstance(v, dict): - cls.merge_update(v, results[k]) - else: - results[k] = v - - @classmethod - def update_parameter(cls, job_id, role, party_id, updated_parameters: dict): - job_configuration = job_utils.get_job_configuration(job_id=job_id, - role=role, - party_id=party_id) - job_parameters = updated_parameters.get("job_parameters") - component_parameters = updated_parameters.get("component_parameters") - if job_parameters: - job_configuration.runtime_conf["job_parameters"] = job_parameters - job_parameters = RunParameters(**job_parameters["common"]) - cls.create_job_parameters_on_party(role=role, - party_id=party_id, - job_parameters=job_parameters) - job_configuration.runtime_conf_on_party["job_parameters"] = job_parameters.to_dict() - if component_parameters: - job_configuration.runtime_conf["component_parameters"] = component_parameters - job_configuration.runtime_conf_on_party["component_parameters"] = component_parameters - - job_info = {} - job_info["job_id"] = job_id - job_info["role"] = role - job_info["party_id"] = party_id - job_info["runtime_conf"] = job_configuration.runtime_conf - job_info["runtime_conf_on_party"] = job_configuration.runtime_conf_on_party - JobSaver.update_job(job_info) - - @classmethod - def initialize_task(cls, role, party_id, task_info: dict): - task_info["role"] = role - task_info["party_id"] = party_id - initialized_result, provider_group = cls.initialize_tasks(components=[task_info["component_name"]], **task_info) - return initialized_result - - @classmethod - def initialize_tasks(cls, job_id, role, party_id, run_on_this_party, initiator_role, initiator_party_id, - job_parameters: RunParameters = None, dsl_parser=None, components: list = None, - runtime_conf=None, check_version=False, is_scheduler=False, **kwargs): - common_task_info = {} - common_task_info["job_id"] = job_id - common_task_info["initiator_role"] = initiator_role - common_task_info["initiator_party_id"] = initiator_party_id - common_task_info["role"] = role - common_task_info["party_id"] = party_id - common_task_info["run_on_this_party"] = run_on_this_party - common_task_info["federated_mode"] = kwargs.get("federated_mode", job_parameters.federated_mode if job_parameters else None) - common_task_info["federated_status_collect_type"] = kwargs.get("federated_status_collect_type", job_parameters.federated_status_collect_type if job_parameters else None) - common_task_info["auto_retries"] = kwargs.get("auto_retries", job_parameters.auto_retries if job_parameters else None) - common_task_info["auto_retry_delay"] = kwargs.get("auto_retry_delay", job_parameters.auto_retry_delay if job_parameters else None) - common_task_info["task_version"] = kwargs.get("task_version") - if role == "local": - common_task_info["run_ip"] = RuntimeConfig.JOB_SERVER_HOST - common_task_info["run_port"] = RuntimeConfig.HTTP_PORT - if dsl_parser is None: - dsl_parser, runtime_conf, dsl = schedule_utils.get_job_dsl_parser_by_job_id(job_id) - provider_group = ProviderManager.get_job_provider_group(dsl_parser=dsl_parser, - runtime_conf=runtime_conf, - components=components, - role=role, - party_id=party_id, - check_version=check_version, - is_scheduler=is_scheduler) - initialized_result = {} - for group_key, group_info in provider_group.items(): - initialized_config = {} - initialized_config.update(group_info) - initialized_config["common_task_info"] = common_task_info - if run_on_this_party: - code, _result = WorkerManager.start_general_worker(worker_name=WorkerName.TASK_INITIALIZER, - job_id=job_id, - role=role, - party_id=party_id, - initialized_config=initialized_config, - run_in_subprocess=False if initialized_config["if_default_provider"] else True) - initialized_result.update(_result) - else: - cls.initialize_task_holder_for_scheduling(role=role, - party_id=party_id, - components=initialized_config["components"], - common_task_info=common_task_info, - provider_info=initialized_config["provider"]) - return initialized_result, provider_group - - @classmethod - def initialize_task_holder_for_scheduling(cls, role, party_id, components, common_task_info, provider_info): - for component_name in components: - task_info = {} - task_info.update(common_task_info) - task_info["component_name"] = component_name - task_info["component_module"] = "" - task_info["provider_info"] = provider_info - task_info["component_parameters"] = {} - TaskController.create_task(role=role, party_id=party_id, - run_on_this_party=common_task_info["run_on_this_party"], - task_info=task_info) - - @classmethod - def initialize_job_tracker(cls, job_id, role, party_id, job_parameters: RunParameters, roles, is_initiator, dsl_parser): - tracker = Tracker(job_id=job_id, role=role, party_id=party_id, - model_id=job_parameters.model_id, - model_version=job_parameters.model_version, - job_parameters=job_parameters) - - partner = {} - show_role = {} - for _role, _role_party in roles.items(): - if is_initiator or _role == role: - show_role[_role] = show_role.get(_role, []) - for _party_id in _role_party: - if is_initiator or _party_id == party_id: - show_role[_role].append(_party_id) - - if _role != role: - partner[_role] = partner.get(_role, []) - partner[_role].extend(_role_party) - else: - for _party_id in _role_party: - if _party_id != party_id: - partner[_role] = partner.get(_role, []) - partner[_role].append(_party_id) - - job_args = dsl_parser.get_args_input() - dataset = get_job_dataset(is_initiator, role, party_id, roles, job_args) - tracker.log_job_view({'partner': partner, 'dataset': dataset, 'roles': show_role}) - - @classmethod - def query_job_input_args(cls, input_data, role, party_id): - min_partition = data_utils.get_input_data_min_partitions( - input_data, role, party_id) - return {'min_input_data_partition': min_partition} - - @classmethod - def align_job_args(cls, job_id, role, party_id, job_info): - job_info["job_id"] = job_id - job_info["role"] = role - job_info["party_id"] = party_id - JobSaver.update_job(job_info) - - @classmethod - def start_job(cls, job_id, role, party_id, extra_info=None): - schedule_logger(job_id).info( - f"try to start job on {role} {party_id}") - job_info = { - "job_id": job_id, - "role": role, - "party_id": party_id, - "status": JobStatus.RUNNING, - "start_time": current_timestamp() - } - if extra_info: - schedule_logger(job_id).info(f"extra info: {extra_info}") - job_info.update(extra_info) - cls.update_job_status(job_info=job_info) - cls.update_job(job_info=job_info) - schedule_logger(job_id).info( - f"start job on {role} {party_id} successfully") - - @classmethod - def update_job(cls, job_info): - """ - Save to local database - :param job_info: - :return: - """ - return JobSaver.update_job(job_info=job_info) - - @classmethod - def update_job_status(cls, job_info): - update_status = JobSaver.update_job_status(job_info=job_info) - if update_status and EndStatus.contains(job_info.get("status")): - ResourceManager.return_job_resource( - job_id=job_info["job_id"], role=job_info["role"], party_id=job_info["party_id"]) - return update_status - - @classmethod - def stop_jobs(cls, job_id, stop_status, role=None, party_id=None): - if role and party_id: - jobs = JobSaver.query_job( - job_id=job_id, role=role, party_id=party_id) - else: - jobs = JobSaver.query_job(job_id=job_id) - kill_status = True - kill_details = {} - for job in jobs: - kill_job_status, kill_job_details = cls.stop_job( - job=job, stop_status=stop_status) - kill_status = kill_status & kill_job_status - kill_details[job_id] = kill_job_details - return kill_status, kill_details - - @classmethod - def stop_job(cls, job, stop_status): - tasks = JobSaver.query_task( - job_id=job.f_job_id, role=job.f_role, party_id=job.f_party_id, only_latest=True, reverse=True) - kill_status = True - kill_details = {} - for task in tasks: - if task.f_status in [TaskStatus.SUCCESS, TaskStatus.WAITING, TaskStatus.PASS]: - continue - kill_task_status = False - status, response = FederatedScheduler.stop_task(job=job, task=task, stop_status=stop_status) - if status == RetCode.SUCCESS: - kill_task_status = True - kill_status = kill_status & kill_task_status - kill_details[task.f_task_id] = 'success' if kill_task_status else 'failed' - if kill_status: - job_info = job.to_human_model_dict(only_primary_with=["status"]) - job_info["status"] = stop_status - JobController.update_job_status(job_info) - return kill_status, kill_details - # Job status depends on the final operation result and initiator calculate - - @classmethod - def save_pipelined_model(cls, job_id, role, party_id): - if role == 'local': - schedule_logger(job_id).info('A job of local role does not need to save pipeline model') - return - - schedule_logger(job_id).info(f'start to save pipeline model on {role} {party_id}') - - job_configuration = job_utils.get_job_configuration(job_id, role, party_id) - - runtime_conf_on_party = job_configuration.runtime_conf_on_party - job_parameters = runtime_conf_on_party['job_parameters'] - - model_id = job_parameters['model_id'] - model_version = job_parameters['model_version'] - job_type = job_parameters.get('job_type', '') - roles = runtime_conf_on_party['role'] - initiator_role = runtime_conf_on_party['initiator']['role'] - initiator_party_id = runtime_conf_on_party['initiator']['party_id'] - assistant_role = job_parameters.get('assistant_role', []) - - if role in set(assistant_role) or job_type == 'predict': - return - - dsl_parser = schedule_utils.get_job_dsl_parser( - dsl=job_configuration.dsl, - runtime_conf=job_configuration.runtime_conf, - train_runtime_conf=job_configuration.train_runtime_conf, - ) - - tasks = JobSaver.query_task( - job_id=job_id, - role=role, - party_id=party_id, - only_latest=True, - ) - components_parameters = { - task.f_component_name: task.f_component_parameters for task in tasks - } - - predict_dsl = schedule_utils.fill_inference_dsl(dsl_parser, job_configuration.dsl, components_parameters) - - pipeline = pipeline_pb2.Pipeline() - - pipeline.roles = json_dumps(roles, byte=True) - - pipeline.model_id = model_id - pipeline.model_version = model_version - - pipeline.initiator_role = initiator_role - pipeline.initiator_party_id = initiator_party_id - - pipeline.train_dsl = json_dumps(job_configuration.dsl, byte=True) - pipeline.train_runtime_conf = json_dumps(job_configuration.runtime_conf, byte=True) - pipeline.runtime_conf_on_party = json_dumps(runtime_conf_on_party, byte=True) - pipeline.inference_dsl = json_dumps(predict_dsl, byte=True) - - pipeline.fate_version = RuntimeConfig.get_env('FATE') - pipeline.parent = True - pipeline.parent_info = json_dumps({}, byte=True) - pipeline.loaded_times = 0 - - tracker = Tracker( - job_id=job_id, role=role, party_id=party_id, - model_id=model_id, model_version=model_version, - job_parameters=RunParameters(**job_parameters), - ) - - if ENABLE_MODEL_STORE: - query = tracker.pipelined_model.pipelined_component.get_define_meta_from_db() - for row in query: - sync_component = SyncComponent( - role=role, party_id=party_id, - model_id=model_id, model_version=model_version, - component_name=row.f_component_name, - ) - if not sync_component.local_exists() and sync_component.remote_exists(): - sync_component.download() - - tracker.pipelined_model.save_pipeline_model(pipeline) - - model_info = gather_model_info_data(tracker.pipelined_model) - save_model_info(model_info) - - schedule_logger(job_id).info(f'save pipeline on {role} {party_id} successfully') - - @classmethod - def clean_job(cls, job_id, role, party_id, roles): - pass - # schedule_logger(job_id).info(f"start to clean job on {role} {party_id}") - # TODO: clean job - # schedule_logger(job_id).info(f"job on {role} {party_id} clean done") - - @classmethod - def job_reload(cls, job): - schedule_logger(job.f_job_id).info(f"start job reload") - cls.log_reload(job) - source_inheritance_tasks, target_inheritance_tasks = cls.load_source_target_tasks(job) - schedule_logger(job.f_job_id).info(f"source_inheritance_tasks:{source_inheritance_tasks}, target_inheritance_tasks:{target_inheritance_tasks}") - cls.output_reload(job, source_inheritance_tasks, target_inheritance_tasks) - if job.f_is_initiator: - source_inheritance_tasks, target_inheritance_tasks = cls.load_source_target_tasks(job, update_status=True) - cls.status_reload(job, source_inheritance_tasks, target_inheritance_tasks) - - @classmethod - def load_source_target_tasks(cls, job, update_status=False): - filters = {"component_list": job.f_inheritance_info.get("component_list", [])} - if not update_status: - filters.update({"role": job.f_role, "party_id": job.f_party_id}) - source_inheritance_tasks = cls.load_tasks(job_id=job.f_inheritance_info.get("job_id"), **filters) - target_inheritance_tasks = cls.load_tasks(job_id=job.f_job_id, **filters) - return source_inheritance_tasks, target_inheritance_tasks - - @classmethod - def load_tasks(cls, component_list, job_id, **kwargs): - tasks = JobSaver.query_task(job_id=job_id, only_latest=True, **kwargs) - task_dict = {} - for cpn in component_list: - for task in tasks: - if cpn == task.f_component_name: - task_dict[f"{cpn}_{task.f_role}_{task.f_task_version}"] = task - return task_dict - - @classmethod - def load_task_tracker(cls, tasks: dict): - tracker_dict = {} - for key, task in tasks.items(): - schedule_logger(task.f_job_id).info( - f"task:{task.f_job_id}, {task.f_role}, {task.f_party_id},{task.f_component_name},{task.f_task_version}") - tracker = Tracker(job_id=task.f_job_id, role=task.f_role, party_id=task.f_party_id, - component_name=task.f_component_name, - task_id=task.f_task_id, - task_version=task.f_task_version) - tracker_dict[key] = tracker - return tracker_dict - - @classmethod - def log_reload(cls, job): - schedule_logger(job.f_job_id).info("start reload job log") - if job.f_inheritance_info: - for component_name in job.f_inheritance_info.get("component_list"): - source_path = os.path.join(log_utils.get_logger_base_dir(), job.f_inheritance_info.get("job_id"), job.f_role, job.f_party_id, component_name) - target_path = os.path.join(log_utils.get_logger_base_dir(), job.f_job_id, job.f_role, job.f_party_id, component_name) - if os.path.exists(source_path): - if os.path.exists(target_path): - shutil.rmtree(target_path) - shutil.copytree(source_path, target_path) - schedule_logger(job.f_job_id).info("reload job log success") - - @classmethod - def output_reload(cls, job, source_tasks: dict, target_tasks: dict): - # model reload - schedule_logger(job.f_job_id).info("start reload model") - source_jobs = JobSaver.query_job(job_id=job.f_inheritance_info["job_id"], role=job.f_role, party_id=job.f_party_id) - if source_jobs: - cls.output_model_reload(job, source_jobs[0]) - schedule_logger(job.f_job_id).info("start reload data") - source_tracker_dict = cls.load_task_tracker(source_tasks) - target_tracker_dict = cls.load_task_tracker(target_tasks) - for key, source_tracker in source_tracker_dict.items(): - target_tracker = target_tracker_dict[key] - table_infos = source_tracker.get_output_data_info() - # data reload - schedule_logger(job.f_job_id).info(f"table infos:{table_infos}") - for table in table_infos: - target_tracker.log_output_data_info(data_name=table.f_data_name, - table_namespace=table.f_table_namespace, - table_name=table.f_table_name) - - # cache reload - schedule_logger(job.f_job_id).info("start reload cache") - cache_list = source_tracker.query_output_cache_record() - for cache in cache_list: - schedule_logger(job.f_job_id).info(f"start reload cache name: {cache.f_cache_name}") - target_tracker.tracking_output_cache(cache.f_cache, cache_name=cache.f_cache_name) - - # summary reload - schedule_logger(job.f_job_id).info("start reload summary") - target_tracker.reload_summary(source_tracker=source_tracker) - - # metric reload - schedule_logger(job.f_job_id).info("start reload metric") - target_tracker.reload_metric(source_tracker=source_tracker) - - schedule_logger(job.f_job_id).info("reload output success") - - @classmethod - def status_reload(cls, job, source_tasks, target_tasks): - schedule_logger(job.f_job_id).info("start reload status") - # update task status - for key, source_task in source_tasks.items(): - try: - JobSaver.reload_task(source_task, target_tasks[key]) - except Exception as e: - schedule_logger(job.f_job_id).warning(f"reload failed: {e}") - - # update job status - JobSaver.update_job(job_info={ - "job_id": job.f_job_id, - "role": job.f_role, - "party_id": job.f_party_id, - "inheritance_status": JobInheritanceStatus.SUCCESS, - }) - schedule_logger(job.f_job_id).info("reload status success") - - @classmethod - def output_model_reload(cls, job, source_job): - source_pipelined_component = PipelinedComponent( - role=source_job.f_role, party_id=source_job.f_party_id, - model_id=source_job.f_runtime_conf['job_parameters']['common']['model_id'], - model_version=source_job.f_job_id, - ) - target_pipelined_component = PipelinedComponent( - role=job.f_role, party_id=job.f_party_id, - model_id=job.f_runtime_conf['job_parameters']['common']['model_id'], - model_version=job.f_job_id, - ) - - query_args = ( - PipelineComponentMeta.f_component_name.in_(job.f_inheritance_info['component_list']), - ) - query = source_pipelined_component.get_define_meta_from_db(*query_args) - - for row in query: - for i in ('variables_data_path', 'run_parameters_path', 'checkpoint_path'): - source_dir = getattr(source_pipelined_component, i) / row.f_component_name - target_dir = getattr(target_pipelined_component, i) / row.f_component_name - - if not source_dir.is_dir(): - continue - if target_dir.is_dir(): - shutil.rmtree(target_dir) - - shutil.copytree(source_dir, target_dir) - - source_pipelined_component.replicate_define_meta({ - 'f_role': target_pipelined_component.role, - 'f_party_id': target_pipelined_component.party_id, - 'f_model_id': target_pipelined_component.model_id, - 'f_model_version': target_pipelined_component.model_version, - }, query_args, True) diff --git a/python/fate_flow/controller/parser.py b/python/fate_flow/controller/parser.py new file mode 100644 index 000000000..87dfc91f4 --- /dev/null +++ b/python/fate_flow/controller/parser.py @@ -0,0 +1,837 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy +import logging +import os +from typing import Dict, Union, List + +import networkx as nx +from pydantic import BaseModel + +from fate_flow.entity.spec.dag import DataWarehouseChannelSpec, ModelWarehouseChannelSpec, \ + RuntimeTaskOutputChannelSpec, ComponentSpec, EggrollComputingSpec, SparkComputingSpec, StandaloneComputingSpec, \ + StandaloneFederationSpec, RollSiteFederationSpec, OSXFederationSpec, \ + PulsarFederationSpec, RabbitMQFederationSpec, FlowLogger, MLMDSpec, TaskRuntimeConfSpec, \ + DAGSchema, DAGSpec, PreTaskConfigSpec, FlowRuntimeInputArtifacts, OutputArtifactType, PartySpec +from fate_flow.entity.types import EngineType, FederationEngine, DataSet, InputArtifactType, ArtifactSourceType, \ + ComputingEngine, OSXMode +from fate_flow.manager.service.provider_manager import ProviderManager +from fate_flow.runtime.job_default_config import JobDefaultConfig +from fate_flow.runtime.system_settings import ENGINES, PROXY, FATE_FLOW_CONF_PATH, HOST, HTTP_PORT, PROTOCOL, \ + API_VERSION, COMPUTING_CONF, LOG_LEVEL +from fate_flow.utils import job_utils, file_utils + + +class TaskNodeInfo(object): + def __init__(self): + self._runtime_parameters = None + self._runtime_parties = None + self._input_dependencies = None + self._component_ref = None + self._component_spec = None + self._upstream_inputs = dict() + self._outputs = dict() + self._stage = None + self._conf = None + + @property + def stage(self): + return self._stage + + @stage.setter + def stage(self, stage): + self._stage = stage + + @property + def runtime_parameters(self): + return self._runtime_parameters + + @runtime_parameters.setter + def runtime_parameters(self, runtime_parameters): + self._runtime_parameters = runtime_parameters + + @property + def runtime_parties(self): + return self._runtime_parties + + @runtime_parties.setter + def runtime_parties(self, runtime_parties): + self._runtime_parties = runtime_parties + + @property + def runtime_roles(self) -> list: + roles = set() + for party_spec in self._runtime_parties: + roles.add(party_spec.role) + + return list(roles) + + @property + def upstream_inputs(self): + return self._upstream_inputs + + @upstream_inputs.setter + def upstream_inputs(self, upstream_inputs): + self._upstream_inputs = upstream_inputs + + @property + def outputs(self): + return self._outputs + + @outputs.setter + def outputs(self, outputs): + self._outputs = outputs + + @property + def component_spec(self): + return self._component_spec + + @component_spec.setter + def component_spec(self, component_spec): + self._component_spec = component_spec + + @property + def output_definitions(self): + return self._component_spec.output_definitions + + @property + def component_ref(self): + return self._component_ref + + @component_ref.setter + def component_ref(self, component_ref): + self._component_ref = component_ref + + @property + def conf(self): + return self._conf + + @conf.setter + def conf(self, conf): + self._conf = conf + + +class TaskParser(object): + def __init__(self, task_node, job_id, task_name, role=None, party_id=None, task_id="", execution_id="", model_id="", + model_version="", task_version=None, parties=None, component=None, provider=None, **kwargs): + self.task_node = task_node + self.model_id = model_id + self.model_version = model_version + self.job_id = job_id + self.task_name = task_name + self.role = role + self.party_id = party_id + self.task_id = task_id + self.task_version = task_version + self.execution_id = execution_id + self.parties = parties + self._provider = None + self.component = component + + @property + def federation_id(self): + return job_utils.generate_task_version_id(task_id=self.task_id, task_version=self.task_version) + + @property + def computing_id(self): + return job_utils.generate_session_id(self.task_id, self.task_version, self.role, self.party_id) + + @property + def runtime_parties(self): + return self.task_node.runtime_parties + + @property + def component_ref(self): + return self.component if self.component else self.task_node.component_ref + + @property + def stage(self): + return self.task_node.stage + + @property + def runtime_parameters(self): + return self.task_node.runtime_parameters + + @property + def output_definitions(self): + return self.task_node.output_definitions + + @property + def task_runtime_conf(self): + _rc = self.task_node.conf + return _rc if _rc else {} + + @property + def task_runtime_launcher(self): + return self.task_runtime_conf.get("launcher_name", "default") + + @property + def engine_run(self): + return self.task_runtime_conf.get("engine_run", {}) + + @property + def provider(self): + if not self._provider: + provider_name = self.task_runtime_conf.get("provider") + self._provider = ProviderManager.check_provider_name(provider_name) + return self._provider + + @property + def timeout(self): + return self.task_runtime_conf.get("timeout", JobDefaultConfig.task_timeout) + + @property + def provider_name(self): + return ProviderManager.parser_provider_name(self.provider)[0] + + @property + def input_parameters(self): + return self.task_node.runtime_parameters + + @staticmethod + def generate_mlmd(): + _type = "flow" + return MLMDSpec( + type=_type, + metadata={ + "host": HOST, + "port": HTTP_PORT, + "protocol": PROTOCOL, + "api_version": API_VERSION + }) + + def generate_logger_conf(self): + task_log_dir = job_utils.get_job_log_directory(self.job_id, self.role, self.party_id, self.task_name) + job_party_log_dir = job_utils.get_job_log_directory(self.job_id, self.role, self.party_id) + delay = True + formatters = None + return FlowLogger.create(task_log_dir=task_log_dir, + job_party_log_dir=job_party_log_dir, + level=logging.getLevelName(LOG_LEVEL), + delay=delay, + formatters=formatters) + + @staticmethod + def generate_device(): + return JobDefaultConfig.task_device + + def generate_computing_conf(self): + if ENGINES.get(EngineType.COMPUTING).lower() == ComputingEngine.STANDALONE: + from fate_flow.runtime.system_settings import STANDALONE_DATA_HOME + return StandaloneComputingSpec( + type=ENGINES.get(EngineType.COMPUTING).lower(), + metadata={"computing_id": self.computing_id, "options": {"data_dir": STANDALONE_DATA_HOME}} + ) + + if ENGINES.get(EngineType.COMPUTING).lower() == ComputingEngine.EGGROLL: + return EggrollComputingSpec( + type=ENGINES.get(EngineType.COMPUTING).lower(), + metadata={ + "computing_id": self.computing_id, + "host": COMPUTING_CONF.get(ComputingEngine.EGGROLL).get("host"), + "port": COMPUTING_CONF.get(ComputingEngine.EGGROLL).get("port") + } + ) + + if ENGINES.get(EngineType.COMPUTING).lower() == ComputingEngine.SPARK: + return SparkComputingSpec( + type=ENGINES.get(EngineType.COMPUTING).lower(), + metadata={"computing_id": self.computing_id} + ) + + @staticmethod + def generate_storage_conf(): + return ENGINES.get(EngineType.STORAGE).lower() + + def generate_federation_conf(self): + parties_info = [] + for party in self.parties: + for _party_id in party.party_id: + parties_info.append({"role": party.role, "partyid": _party_id}) + parties = { + "local": {"role": self.role, "partyid": self.party_id}, + "parties": parties_info + } + engine_name = ENGINES.get(EngineType.FEDERATION).lower() + proxy_conf = copy.deepcopy(PROXY.get(engine_name, {})) + if engine_name == FederationEngine.STANDALONE: + spec = StandaloneFederationSpec(type=engine_name, metadata=StandaloneFederationSpec.MetadataSpec( + federation_id=self.federation_id, parties=parties)) + elif engine_name == FederationEngine.ROLLSITE: + spec = RollSiteFederationSpec(type=engine_name, metadata=RollSiteFederationSpec.MetadataSpec( + federation_id=self.federation_id, + parties=parties, + rollsite_config=RollSiteFederationSpec.MetadataSpec.RollSiteConfig(**proxy_conf) + )) + elif engine_name == FederationEngine.OSX: + mode = proxy_conf.pop("mode", OSXMode.QUEUE) + if mode == OSXMode.QUEUE: + spec = OSXFederationSpec(type=engine_name, metadata=OSXFederationSpec.MetadataSpec( + federation_id=self.federation_id, + parties=parties, + osx_config=OSXFederationSpec.MetadataSpec.OSXConfig(**proxy_conf) + )) + elif mode == OSXMode.STREAM: + spec = RollSiteFederationSpec( + type=FederationEngine.ROLLSITE, + metadata=RollSiteFederationSpec.MetadataSpec( + federation_id=self.federation_id, + parties=parties, + rollsite_config=RollSiteFederationSpec.MetadataSpec.RollSiteConfig(**proxy_conf) + )) + else: + raise RuntimeError(f"federation engine {engine_name} mode {mode}is not supported") + elif engine_name == FederationEngine.PULSAR: + route_table_path = os.path.join(FATE_FLOW_CONF_PATH, "pulsar_route_table.yaml") + route_table = file_utils.load_yaml_conf(conf_path=route_table_path) + + spec = PulsarFederationSpec(type=engine_name, metadata=PulsarFederationSpec.MetadataSpec( + federation_id=self.federation_id, + parties=parties, + route_table=PulsarFederationSpec.MetadataSpec.RouteTable( + route={k: PulsarFederationSpec.MetadataSpec.RouteTable.Route(**v) for k, v in route_table.items() if + k != "default"}, + default=PulsarFederationSpec.MetadataSpec.RouteTable.Default( + **route_table.get("default", {})) if route_table.get("default") else None + ), + pulsar_config=PulsarFederationSpec.MetadataSpec.PulsarConfig(**proxy_conf) + )) + elif engine_name == FederationEngine.RABBITMQ: + route_table_path = os.path.join(FATE_FLOW_CONF_PATH, "rabbitmq_route_table.yaml") + route_table = file_utils.load_yaml_conf(conf_path=route_table_path) + spec = RabbitMQFederationSpec(type=engine_name, metadata=RabbitMQFederationSpec.MetadataSpec( + federation_id=self.federation_id, + parties=parties, + route_table={k: RabbitMQFederationSpec.MetadataSpec.RouteTable(**v) for k, v in route_table.items()}, + rabbitmq_config=RabbitMQFederationSpec.MetadataSpec.RabbitMQConfig(**proxy_conf) + )) + else: + raise RuntimeError(f"federation engine {engine_name} is not supported") + return spec + + @property + def task_conf(self): + return TaskRuntimeConfSpec( + logger=self.generate_logger_conf(), + device=self.generate_device(), + computing=self.generate_computing_conf(), + federation=self.generate_federation_conf(), + storage=self.generate_storage_conf() + ) + + @property + def task_parameters(self) -> PreTaskConfigSpec: + return PreTaskConfigSpec( + model_id=self.model_id, + model_version=self.model_version, + job_id=self.job_id, + task_id=self.task_id, + task_version=self.task_version, + task_name=self.task_name, + provider_name=self.provider_name, + party_task_id=self.execution_id, + component=self.component_ref, + role=self.role, + stage=self.stage, + party_id=self.party_id, + parameters=self.input_parameters, + input_artifacts=self.task_node.upstream_inputs, + conf=self.task_conf, + mlmd=self.generate_mlmd() + ) + + +class DagParser(object): + def __init__(self): + self._dag = dict() + self._global_dag = nx.DiGraph() + self._links = dict() + self._task_parameters = dict() + self._task_parties = dict() + self._tasks = dict() + self._task_runtime_parties = dict() + self._conf = dict() + + def parse_dag(self, dag_schema: DAGSchema, component_specs: Dict[str, ComponentSpec] = None): + dag_spec = dag_schema.dag + dag_stage = dag_spec.stage + tasks = dag_spec.tasks + if dag_spec.conf: + self._conf = dag_spec.conf.dict(exclude_defaults=True) + job_conf = self._conf.get("task", {}) + + for party in dag_spec.parties: + if party.role not in self._dag: + self._dag[party.role] = dict() + for party_id in party.party_id: + self._dag[party.role][party_id] = nx.DiGraph() + + for name, task_spec in tasks.items(): + parties = task_spec.parties if task_spec.parties else dag_spec.parties + task_stage = dag_stage + component_ref = task_spec.component_ref + if task_spec.stage: + task_stage = task_spec.stage + + self._global_dag.add_node(name, component_ref=component_ref) + + self._task_runtime_parties[name] = parties + + for party_spec in parties: + if party_spec.role not in self._tasks: + self._tasks[party_spec.role] = dict() + for party_id in party_spec.party_id: + self._dag[party_spec.role][party_id].add_node(name) + if party_id not in self._tasks[party_spec.role]: + self._tasks[party_spec.role][party_id] = dict() + self._tasks[party_spec.role][party_id].update({ + name: TaskNodeInfo() + }) + self._tasks[party_spec.role][party_id][name].stage = task_stage + self._tasks[party_spec.role][party_id][name].component_ref = component_ref + if component_specs: + self._tasks[party_spec.role][party_id][name].component_spec = component_specs[name] + + for name, task_spec in tasks.items(): + if not task_spec.conf: + task_conf = copy.deepcopy(job_conf) + else: + task_conf = copy.deepcopy(job_conf) + task_conf.update(task_spec.conf) + + self._init_task_runtime_parameters_and_conf(name, dag_schema, task_conf) + + self._init_upstream_inputs(name, dag_schema.dag) + self._init_outputs(name, dag_schema.dag) + + def _init_upstream_inputs(self, name, dag: DAGSpec): + task_spec = dag.tasks[name] + upstream_inputs = dict() + + parties = task_spec.parties if task_spec.parties else dag.parties + for party in parties: + if party.role not in upstream_inputs: + upstream_inputs[party.role] = dict() + for party_id in party.party_id: + self._tasks[party.role][party_id][name].upstream_inputs = self._get_upstream_inputs( + name, task_spec, party.role, party_id + ) + + def _get_upstream_inputs(self, name, task_spec, role, party_id): + upstream_inputs = dict() + runtime_parties = task_spec.parties + + if runtime_parties: + runtime_parties_dict = dict((party.role, party.party_id) for party in runtime_parties) + if role not in runtime_parties_dict or party_id not in runtime_parties_dict[role]: + return upstream_inputs + + input_artifacts = task_spec.inputs + + if not input_artifacts: + return upstream_inputs + + for input_type in InputArtifactType.types(): + artifacts = getattr(input_artifacts, input_type) + if not artifacts: + continue + + for input_key, output_specs_dict in artifacts.items(): + for artifact_source, channel_spec_list in output_specs_dict.items(): + if artifact_source == ArtifactSourceType.MODEL_WAREHOUSE: + is_list = True + if not isinstance(channel_spec_list, list): + is_list = False + channel_spec_list = [channel_spec_list] + inputs = [] + for channel in channel_spec_list: + model_warehouse_channel = ModelWarehouseChannelSpec(**channel.dict(exclude_defaults=True)) + if model_warehouse_channel.parties and not self.task_can_run( + role, party_id, runtime_parties=model_warehouse_channel.parties): + continue + + if model_warehouse_channel.model_id is None: + model_warehouse_channel.model_id = \ + self._conf.get("model_warehouse", {}).get("model_id", None) + model_warehouse_channel.model_version = \ + self._conf.get("model_warehouse", {}).get("model_version", None) + inputs.append(model_warehouse_channel) + + if not inputs: + continue + + if input_type not in upstream_inputs: + upstream_inputs[input_type] = dict() + + if is_list and len(inputs) == 1: + is_list = False + upstream_inputs[input_type][input_key] = inputs if is_list else inputs[0] + elif artifact_source == ArtifactSourceType.DATA_WAREHOUSE: + is_list = True + if not isinstance(channel_spec_list, list): + is_list = False + channel_spec_list = [channel_spec_list] + inputs = [] + for channel in channel_spec_list: + if channel.parties and \ + not self.task_can_run(role, party_id, runtime_parties=channel.parties): + continue + inputs.append(DataWarehouseChannelSpec(**channel.dict(exclude_defaults=True))) + + if not inputs: + continue + if input_type not in upstream_inputs: + upstream_inputs[input_type] = dict() + + if is_list and len(inputs) == 1: + is_list = False + upstream_inputs[input_type][input_key] = inputs if is_list else inputs[0] + else: + if not isinstance(channel_spec_list, list): + channel_spec_list = [channel_spec_list] + + filter_channel_spec_list = [] + for channel_spec in channel_spec_list: + if channel_spec.parties: + parties_dict = dict((party.role, party.party_id) for party in channel_spec.parties) + if role not in parties_dict or party_id not in parties_dict[role]: + continue + else: + if channel_spec.producer_task not in self._dag[role][party_id].nodes: + continue + filter_channel_spec_list.append(channel_spec) + + if not filter_channel_spec_list: + continue + + if len(filter_channel_spec_list) > 1: + inputs = [RuntimeTaskOutputChannelSpec(**channel.dict(exclude_defaults=True)) + for channel in filter_channel_spec_list] + else: + inputs = RuntimeTaskOutputChannelSpec(**filter_channel_spec_list[0].dict(exclude_defaults=True)) + + if not inputs: + continue + + if input_type not in upstream_inputs: + upstream_inputs[input_type] = dict() + upstream_inputs[input_type][input_key] = inputs + + for channel_spec in filter_channel_spec_list: + dependent_task = channel_spec.producer_task + self._add_edge(dependent_task, name, role, party_id) + + upstream_inputs = self.check_and_add_runtime_party(upstream_inputs, role, party_id, artifact_type="input") + + return upstream_inputs + + def _init_outputs(self, name, dag: DAGSpec): + task_spec = dag.tasks[name] + + if not task_spec.outputs: + return + + parties = task_spec.parties if task_spec.parties else dag.parties + + for output_type, outputs_dict in iter(task_spec.outputs): + if not outputs_dict: + continue + + for outputs_key, output_artifact in outputs_dict.items(): + output_parties = output_artifact.parties if output_artifact.parties else parties + for party_spec in output_parties: + for party_id in party_spec.party_id: + if not self.task_can_run(party_spec.role, party_id, runtime_parties=parties): + continue + + if outputs_key not in self._tasks[party_spec.role][party_id][name].outputs: + self._tasks[party_spec.role][party_id][name].outputs[output_type] = dict() + + self._tasks[party_spec.role][party_id][name].outputs[output_type][outputs_key] = output_artifact + + for party_spec in parties: + for party_id in party_spec.party_id: + self._tasks[party_spec.role][party_id][name].outputs = self.check_and_add_runtime_party( + self._tasks[party_spec.role][party_id][name].outputs, + party_spec.role, + party_id, + artifact_type="output" + ) + + def _add_edge(self, src, dst, role, party_id, attrs=None): + if not attrs: + attrs = {} + + self._dag[role][party_id].add_edge(src, dst, **attrs) + self._global_dag.add_edge(src, dst, **attrs) + + def _init_task_runtime_parameters_and_conf(self, task_name: str, dag_schema: DAGSchema, global_task_conf): + dag = dag_schema.dag + task_spec = dag.tasks[task_name] + + common_parameters = dict() + if task_spec.parameters: + common_parameters = task_spec.parameters + + parties = dag.parties if not task_spec.parties else task_spec.parties + + for party in parties: + for party_id in party.party_id: + self._tasks[party.role][party_id][task_name].runtime_parameters = copy.deepcopy(common_parameters) + self._tasks[party.role][party_id][task_name].conf = copy.deepcopy(global_task_conf) + + if dag.party_tasks: + party_tasks = dag.party_tasks + for site_name, party_tasks_spec in party_tasks.items(): + if party_tasks_spec.conf: + for party in party_tasks_spec.parties: + for party_id in party.party_id: + self._tasks[party.role][party_id][task_name].conf.update(party_tasks_spec.conf) + + if not party_tasks_spec.tasks or task_name not in party_tasks_spec.tasks: + continue + + party_parties = party_tasks_spec.parties + party_task_spec = party_tasks_spec.tasks[task_name] + + if party_task_spec.conf: + for party in party_parties: + for party_id in party.party_id: + self._tasks[party.role][party_id][task_name].conf.update(party_task_spec.conf) + + parameters = party_task_spec.parameters + + if parameters: + for party in party_parties: + for party_id in party.party_id: + self._tasks[party.role][party_id][task_name].runtime_parameters.update(parameters) + + def get_runtime_roles_on_party(self, task_name, party_id): + task_runtime_parties = self._task_runtime_parties[task_name] + + runtime_roles = set() + for party_spec in task_runtime_parties: + if party_id in party_spec.party_id: + runtime_roles.add(party_spec.role) + + return list(runtime_roles) + + def get_task_node(self, role, party_id, task_name): + if role not in self._tasks: + raise ValueError(f"role={role} does ont exist in dag") + if party_id not in self._tasks[role]: + raise ValueError(f"role={role}, party_id={party_id} does not exist in dag") + if task_name not in self._tasks[role][party_id]: + raise ValueError(f"role={role}, party_id={party_id} does not has task {task_name}") + + return self._tasks[role][party_id][task_name] + + def get_need_revisit_tasks(self, visited_tasks, failed_tasks, role, party_id): + """ + visited_tasks: already visited tasks + failed_tasks: failed tasks + + this function finds tasks need to rerun, a task need to rerun if is upstreams is failed + """ + invalid_tasks = set(self.party_topological_sort(role, party_id)) - set(visited_tasks) + invalid_tasks |= set(failed_tasks) + + revisit_tasks = [] + for task_to_check in visited_tasks: + if task_to_check in invalid_tasks: + revisit_tasks.append(task_to_check) + continue + + task_valid = True + task_stack = {task_to_check} + stack = [task_to_check] + + while len(stack) > 0 and task_valid: + task = stack.pop() + pre_tasks = self.party_predecessors(role, party_id, task) + + for pre_task in pre_tasks: + if pre_task in task_stack: + continue + if pre_task in invalid_tasks: + task_valid = False + break + + task_stack.add(pre_task) + stack.append(pre_task) + + if not task_valid: + revisit_tasks.append(task_to_check) + + return revisit_tasks + + def topological_sort(self, role, party_id): + return nx.topological_sort(self._dag[role][party_id]) + + def global_topological_sort(self): + return nx.topological_sort(self._global_dag) + + def get_component_ref(self, task_name): + return self._global_dag.nodes[task_name]["component_ref"] + + def party_topological_sort(self, role, party_id): + assert role in self._dag or party_id in self._dag[role], f"role={role}, party_id={party_id} does not exist" + return nx.topological_sort(self._dag[role][party_id]) + + def party_predecessors(self, role, party_id, task): + return set(self._dag[role][party_id].predecessors(task)) + + def party_successors(self, role, party_id, task): + return self._dag[role][party_id].successors(task) + + def get_edge_attr(self, role, party_id, src, dst): + return self._dag[role][party_id].edges[src, dst] + + @classmethod + def task_can_run(cls, role, party_id, component_spec: ComponentSpec=None, runtime_parties: List[PartySpec]=None): + if component_spec and role not in component_spec.roles: + return False + + for party_spec in runtime_parties: + if role == party_spec.role and party_id in party_spec.party_id: + return True + + return False + + @staticmethod + def check_and_add_runtime_party(artifacts, role, party_id, artifact_type): + correct_artifacts = copy.deepcopy(artifacts) + if artifact_type == "input": + types = InputArtifactType.types() + else: + types = OutputArtifactType.types() + + for t in types: + if t not in artifacts: + continue + for _key, channel_list in artifacts[t].items(): + if isinstance(channel_list, list): + for idx, channel in enumerate(channel_list): + correct_artifacts[t][_key][idx].parties = [PartySpec(role=role, party_id=[party_id])] + else: + correct_artifacts[t][_key].parties = [PartySpec(role=role, party_id=[party_id])] + + return correct_artifacts + + @property + def conf(self): + return self._conf + + @property + def task_runtime_parties(self): + return self._task_runtime_parties + + def get_task_runtime_parties(self, task_name): + return self._task_runtime_parties[task_name] + + @classmethod + def infer_dependent_tasks(cls, input_artifacts): + if not input_artifacts: + return [] + + dependent_task_list = list() + for input_type in InputArtifactType.types(): + artifacts = getattr(input_artifacts, input_type) + if not artifacts: + continue + for artifact_name, artifact_channel in artifacts.items(): + for artifact_source_type, channels in artifact_channel.items(): + if artifact_source_type in [ArtifactSourceType.MODEL_WAREHOUSE, ArtifactSourceType.DATA_WAREHOUSE]: + continue + + if not isinstance(channels, list): + channels = [channels] + for channel in channels: + dependent_task_list.append(channel.producer_task) + + return dependent_task_list + + @classmethod + def translate_dag(cls, src, dst, adapter_map, *args, **kwargs): + translate_func = adapter_map[src][dst] + return translate_func(*args, **kwargs) + + +class JobParser(object): + def __init__(self, dag_conf): + self.dag_parser = DagParser() + self.dag_parser.parse_dag(dag_conf) + + def get_task_node(self, role, party_id, task_name): + return self.dag_parser.get_task_node(role, party_id, task_name) + + def topological_sort(self): + return self.dag_parser.global_topological_sort() + + def global_topological_sort(self): + return self.dag_parser.global_topological_sort() + + def party_topological_sort(self, role, party_id): + return self.dag_parser.party_topological_sort(role, party_id) + + def infer_dependent_tasks(self, input_artifacts): + return self.dag_parser.infer_dependent_tasks(input_artifacts) + + @property + def task_parser(self): + return TaskParser + + def component_ref_list(self, role, party_id): + _list = [] + for name in self.party_topological_sort(role=role, party_id=party_id): + node = self.get_task_node(role=role, party_id=party_id, task_name=name) + if node: + _list.append(node.component_ref) + return _list + + def dataset_list(self, role, party_id): + data_set = [] + for task_name in self.party_topological_sort(role=role, party_id=party_id): + task_node = self.get_task_node(role=role, party_id=party_id, task_name=task_name) + parties = self.get_task_runtime_parties(task_name=task_name) + if task_node.component_ref.lower() == "reader" and job_utils.check_party_in(role, party_id, parties): + name = task_node.runtime_parameters.get("name") + namespace = task_node.runtime_parameters.get("namespace") + data_set.append(DataSet(**{"name": name, "namespace": namespace})) + return data_set + + def role_parameters(self, role, party_id): + _dict = {} + for task_name in self.party_topological_sort(role=role, party_id=party_id): + task_node = self.get_task_node(task_name=task_name, role=role, party_id=party_id) + _dict[task_node.component_ref] = task_node.runtime_parameters + return _dict + + def get_runtime_roles_on_party(self, task_name, party_id): + return self.dag_parser.get_runtime_roles_on_party(task_name, party_id) + + def get_task_runtime_parties(self, task_name): + try: + return self.dag_parser.get_task_runtime_parties(task_name) + except: + return [] + + def get_component_ref(self, task_name): + return self.dag_parser.get_component_ref(task_name) + + +class Party(BaseModel): + role: str + party_id: Union[str, int] diff --git a/python/fate_flow/controller/permission.py b/python/fate_flow/controller/permission.py new file mode 100644 index 000000000..745aa8bcc --- /dev/null +++ b/python/fate_flow/controller/permission.py @@ -0,0 +1,256 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import hashlib +import time + +from fate_flow.db.casbin_models import FATE_CASBIN, PERMISSION_CASBIN as PC +from fate_flow.errors.server_error import NoPermission, PermissionOperateError +from fate_flow.manager.service.provider_manager import ProviderManager +from fate_flow.utils.log_utils import getLogger +from fate_flow.entity.types import PermissionParameters, DataSet, PermissionType +from fate_flow.hook.common.parameters import PermissionReturn +from fate_flow.errors.server_error import RequestExpired, NoFoundAppid, InvalidParameter, RoleTypeError +from fate_flow.manager.service.app_manager import AppManager +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.runtime.system_settings import CLIENT_AUTHENTICATION, SITE_AUTHENTICATION +from fate_flow.utils.base_utils import generate_random_id +from fate_flow.utils.wraps_utils import switch_function, check_permission + +logger = getLogger("permission") + + +class Authentication(object): + @classmethod + def md5_sign(cls, app_id, app_token, user_name, initiator_party_id, timestamp, nonce): + key = hashlib.md5(str(app_id + user_name + initiator_party_id + nonce + timestamp).encode("utf8")).hexdigest().lower() + sign = hashlib.md5(str(key + app_token).encode("utf8")).hexdigest().lower() + return sign + + @classmethod + def md5_verify(cls, app_id, timestamp, nonce, signature, user_name="", initiator_party_id=""): + if cls.check_if_expired(timestamp): + raise RequestExpired() + apps = AppManager.query_app(app_id=app_id) + if apps: + _signature = cls.md5_sign( + app_id=app_id, + app_token=apps[0].f_app_token, + user_name=user_name, + initiator_party_id=initiator_party_id, + timestamp=timestamp, + nonce=nonce + ) + return _signature == signature + else: + raise NoFoundAppid(app_id=app_id) + + @staticmethod + def generate_timestamp(): + return str(int(time.time()*1000)) + + @staticmethod + def generate_nonce(): + return generate_random_id(length=4, only_number=True) + + @staticmethod + def check_if_expired(timestamp, timeout=60): + expiration = int(timestamp) + timeout * 1000 + if expiration < int(time.time() * 1000): + return True + else: + return False + + +class PermissionController(object): + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + def add_policy(role, resource, permission): + return FATE_CASBIN.add_policy(role, resource, permission) + + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @AppManager.check_app_id + @check_permission(operate="grant", types="permission") + @AppManager.check_app_type + def add_role_for_user(app_id, role, init=False): + PermissionController.check_permission_role(role) + return FATE_CASBIN.add_role_for_user(app_id, role) + + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @AppManager.check_app_id + @check_permission(operate="delete", types="permission") + # @AppManager.check_app_type + def delete_role_for_user(app_id, role, grant_role=None, init=False): + role_type = role + PermissionController.check_permission_role(role) + app_info = AppManager.query_app(app_id=app_id) + if grant_role == "super_client": + grant_role = "client" + if grant_role and grant_role != app_info[0].f_app_type: + raise RoleTypeError(role=grant_role) + return FATE_CASBIN.delete_role_for_suer(app_id, role_type) + + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @AppManager.check_app_id + @check_permission(operate="query", types="permission") + def get_roles_for_user(app_id): + return FATE_CASBIN.get_roles_for_user(app_id) + + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + def get_permissions_for_user(app_id): + return FATE_CASBIN.get_permissions_for_user(app_id) + + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @AppManager.check_app_id + def delete_roles_for_user(app_id): + return FATE_CASBIN.delete_roles_for_user(app_id) + + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @AppManager.check_app_id + def has_role_for_user(app_id, role): + return FATE_CASBIN.has_role_for_user(app_id, role) + + @staticmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @AppManager.check_app_id + def enforcer(app_id, resource, permission): + return FATE_CASBIN.enforcer(app_id, resource, permission) + + @staticmethod + def check_permission_role(role): + if role not in RuntimeConfig.CLIENT_ROLE: + raise InvalidParameter(role=role) + + +class ResourcePermissionController: + def __init__(self, party_id): + self.party_id = party_id + self.casbin_controller = PC + if not self.casbin_controller: + raise PermissionOperateError(message="No permission controller is found") + + def check(self, permission_type, value): + logger.info(f"check source party id {self.party_id} {permission_type} {value}") + result = self.casbin_controller.enforce(self.party_id, permission_type, value) + logger.info(f"result: {result}") + return result + + def grant_or_delete(self, permission_parameters: PermissionParameters): + logger.info(f"{'grant' if not permission_parameters.is_delete else 'delete'} parameters:" + f" {permission_parameters.to_dict()}") + self.check_parameters(permission_parameters) + for permission_type in PermissionType.values(): + permission_value = getattr(permission_parameters, permission_type) + if permission_value: + if permission_value != "*": + if permission_type in [PermissionType.COMPONENT.value]: + value_list = [value.strip() for value in permission_value.split(self.value_delimiter)] + elif permission_type in [PermissionType.DATASET.value]: + if isinstance(permission_value, list): + value_list = [DataSet(**value).casbin_value for value in permission_value] + else: + value_list = [DataSet(**permission_value).casbin_value] + else: + raise PermissionOperateError(type=permission_type, message="Not Supported") + for value in value_list: + if not permission_parameters.is_delete: + self.casbin_controller.grant(self.party_id, permission_type, value) + else: + self.casbin_controller.delete(self.party_id, permission_type, value) + else: + if not permission_parameters.is_delete: + for value in self.all_value(permission_type): + self.casbin_controller.grant(self.party_id, permission_type, value) + else: + self.casbin_controller.delete_all(self.party_id, permission_type) + + def query(self): + result = {PermissionType.DATASET.value: [], PermissionType.COMPONENT.value: []} + for casbin_result in self.casbin_controller.query(self.party_id): + if casbin_result[1] == PermissionType.DATASET.value: + casbin_result[2] = DataSet.load_casbin_value(casbin_result[2]) + result[casbin_result[1]].append(casbin_result[2]) + return result + + def check_parameters(self, permission_parameters): + for permission_type in PermissionType.values(): + permission_value = getattr(permission_parameters, permission_type) + if permission_value: + if permission_type in [PermissionType.COMPONENT.value]: + if permission_value != "*": + value_list = [value.strip() for value in permission_value.split(self.value_delimiter)] + self.check_values(permission_type, value_list) + if permission_type in [PermissionType.DATASET.value]: + if isinstance(permission_value, list): + for dataset in permission_value: + DataSet(**dataset).check() + elif isinstance(permission_value, dict): + DataSet(**permission_value).check() + elif permission_value == "*": + pass + else: + raise PermissionOperateError(type=permission_type, value=permission_value) + + def check_values(self, permission_type, values): + error_value = [] + value_list = self.all_value(permission_type) + for value in values: + if value not in value_list: + error_value.append(value) + if error_value: + raise PermissionOperateError(type=permission_type, value=error_value) + + def all_value(self, permission_type): + if permission_type == PermissionType.COMPONENT.value: + value_list = self.all_component + else: + raise PermissionOperateError(type=permission_type, message="Not Support Grant all") + return value_list + + @property + def all_component(self): + return ProviderManager.get_all_components() + + @property + def value_delimiter(self): + return "," + + +class PermissionCheck(object): + def __init__(self, party_id, component_list, dataset_list, **kwargs): + self.component_list = component_list + self.dataset_list = dataset_list + self.controller = ResourcePermissionController(party_id) + + def check_component(self) -> PermissionReturn: + for component_name in self.component_list: + if not self.controller.check(PermissionType.COMPONENT.value, component_name): + e = NoPermission(type=PermissionType.COMPONENT.value, component_name=component_name) + return PermissionReturn(code=e.code, message=e.message) + return PermissionReturn() + + def check_dataset(self) -> PermissionReturn: + for dataset in self.dataset_list: + if not self.controller.check(PermissionType.DATASET.value, dataset.casbin_value): + e = NoPermission(type=PermissionType.DATASET.value, dataset=dataset.value) + return PermissionReturn(e.code, e.message) + return PermissionReturn() diff --git a/python/fate_flow/controller/permission_controller.py b/python/fate_flow/controller/permission_controller.py deleted file mode 100644 index 8c1b53618..000000000 --- a/python/fate_flow/controller/permission_controller.py +++ /dev/null @@ -1,130 +0,0 @@ -from fate_flow.db.fate_casbin import CB -from fate_flow.utils.log_utils import getLogger -from fate_flow.entity.permission_parameters import PermissionParameters, DataSet, CheckReturn -from fate_flow.entity.types import PermissionType -from fate_flow.hook.common.parameters import PermissionReturn -from fate_flow.settings import DATASET_PERMISSION, COMPONENT_PERMISSION, CASBIN_MODEL_CONF - -logger = getLogger("permission") - - -class PermissionController: - def __init__(self, src_party_id): - self.src_party_id = str(src_party_id) - self.casbin_controller = CB - if not self.casbin_controller: - raise Exception("No permission controller is found, please check whether the switch of permission control" - " is turned on") - - def check(self, permission_type, value): - logger.info(f"check source party id {self.src_party_id} {permission_type} {value}") - result = self.casbin_controller.enforce(self.src_party_id, permission_type, value) - logger.info(f"result: {result}") - return result - - def grant_or_delete(self, permission_parameters: PermissionParameters): - logger.info(f"{'grant' if not permission_parameters.is_delete else 'delete'} parameters:" - f" {permission_parameters.to_dict()}") - self.check_parameters(permission_parameters) - for permission_type in PermissionType.values(): - permission_value = getattr(permission_parameters, permission_type) - if permission_value: - if permission_value != "*": - if permission_type in [PermissionType.COMPONENT.value]: - value_list = [value.strip() for value in permission_value.split(self.value_delimiter)] - elif permission_type in [PermissionType.DATASET.value]: - if isinstance(permission_value, list): - value_list = [DataSet(**value).casbin_value for value in permission_value] - else: - value_list = [DataSet(**permission_value).casbin_value] - else: - raise ValueError(f"permission type {permission_type} is not supported") - for value in value_list: - if not permission_parameters.is_delete: - self.casbin_controller.grant(self.src_party_id, permission_type, value) - else: - self.casbin_controller.delete(self.src_party_id, permission_type, value) - else: - if not permission_parameters.is_delete: - for value in self.all_value(permission_type): - self.casbin_controller.grant(self.src_party_id, permission_type, value) - else: - self.casbin_controller.delete_all(self.src_party_id, permission_type) - - def query(self): - result = {PermissionType.DATASET.value: [], PermissionType.COMPONENT.value: []} - for casbin_result in self.casbin_controller.query(self.src_party_id): - if casbin_result[1] == PermissionType.DATASET.value: - casbin_result[2] = DataSet.load_casbin_value(casbin_result[2]) - result[casbin_result[1]].append(casbin_result[2]) - return result - - def check_parameters(self, permission_parameters): - for permission_type in PermissionType.values(): - permission_value = getattr(permission_parameters, permission_type) - if permission_value: - if permission_type == PermissionType.COMPONENT.value and not COMPONENT_PERMISSION: - raise ValueError(f"component permission switch is {COMPONENT_PERMISSION}") - if permission_type == PermissionType.DATASET.value and not DATASET_PERMISSION: - raise ValueError(f"dataset permission switch is {DATASET_PERMISSION}") - if permission_type in [PermissionType.COMPONENT.value]: - if permission_value != "*": - value_list = [value.strip() for value in permission_value.split(self.value_delimiter)] - self.check_values(permission_type, value_list) - if permission_type in [PermissionType.DATASET.value]: - if isinstance(permission_value, list): - for dataset in permission_value: - DataSet(**dataset).check() - elif isinstance(permission_value, dict): - DataSet(**permission_value).check() - elif permission_value == "*": - pass - else: - raise ValueError(f"permission type {permission_type} value {permission_value} error") - - def check_values(self, permission_type, values): - error_value = [] - value_list = self.all_value(permission_type) - for value in values: - if value not in value_list: - error_value.append(value) - if error_value: - raise ValueError(f"permission type {permission_type} value {error_value} error") - - def all_value(self, permission_type): - if permission_type == PermissionType.COMPONENT.value: - value_list = self.all_component - else: - raise Exception(f"permission type {permission_type} not support grant all") - return value_list - - @property - def all_component(self): - from fate_flow.db.db_models import ComponentInfo - component_list = [] - for component in ComponentInfo.select(): - component_list.append(component.f_component_name.lower()) - return component_list - - @property - def value_delimiter(self): - return "," - - -class PermissionCheck(object): - def __init__(self, src_party_id, component_list, dataset_list, **kwargs): - self.component_list = component_list - self.dataset_list = dataset_list - self.controller = PermissionController(src_party_id) - - def check_component(self) -> PermissionReturn: - for component_name in self.component_list: - if not self.controller.check(PermissionType.COMPONENT.value, component_name): - return PermissionReturn(CheckReturn.NO_COMPONENT_PERMISSION, f"check component permission failed: {component_name}") - return PermissionReturn() - - def check_dataset(self) -> PermissionReturn: - for dataset in self.dataset_list: - if not self.controller.check(PermissionType.DATASET.value, dataset.casbin_value): - return PermissionReturn(CheckReturn.NO_DATASET_PERMISSION, f"check dataset permission failed: {dataset.value}") - return PermissionReturn() \ No newline at end of file diff --git a/python/fate_flow/controller/task.py b/python/fate_flow/controller/task.py new file mode 100644 index 000000000..3c3453f2f --- /dev/null +++ b/python/fate_flow/controller/task.py @@ -0,0 +1,402 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import copy +import os + +import yaml + +from fate_flow.controller.parser import JobParser +from fate_flow.db.db_models import Task +from fate_flow.db.schedule_models import ScheduleTask, ScheduleJob, ScheduleTaskStatus +from fate_flow.engine.devices import build_engine, EngineABC +from fate_flow.entity.spec.dag import DAGSchema +from fate_flow.manager.service.resource_manager import ResourceManager +from fate_flow.manager.service.worker_manager import WorkerManager +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.controller.federated import FederatedScheduler +from fate_flow.entity.types import EndStatus, TaskStatus, FederatedCommunicationType, LauncherType +from fate_flow.entity.code import FederatedSchedulingStatusCode +from fate_flow.manager.operation.job_saver import JobSaver, ScheduleJobSaver +from fate_flow.utils import job_utils +from fate_flow.utils.base_utils import current_timestamp +from fate_flow.utils.log_utils import schedule_logger +from fate_flow.utils.wraps_utils import asynchronous_function + + +class TaskController(object): + INITIATOR_COLLECT_FIELDS = ["status", "party_status", "start_time", "update_time", "end_time", "elapsed"] + + @classmethod + def create_tasks(cls, job_id: str, role: str, party_id: str, dag_schema: DAGSchema, task_run=None, task_cores=None, + is_scheduler=False): + schedule_logger(job_id).info(f"start create {'scheduler' if is_scheduler else 'partner'} tasks ...") + job_parser = JobParser(dag_schema) + task_list = job_parser.global_topological_sort() + for task_name in task_list: + parties = job_parser.get_task_runtime_parties(task_name=task_name) + need_run = job_utils.check_party_in(role, party_id, parties) + schedule_logger(job_id).info(f"task {task_name} role {role} party id {party_id} need run status {need_run}") + if need_run: + cls.create_task(job_id, role, party_id, task_name, dag_schema, job_parser, task_run=task_run, + is_scheduler=is_scheduler, task_cores=task_cores) + schedule_logger(job_id).info("create tasks success") + + @classmethod + def create_task(cls, job_id, role, party_id, task_name, dag_schema, job_parser, is_scheduler, task_run=None, + task_cores=None, task_version=0): + task_id = job_utils.generate_task_id(job_id=job_id, component_name=task_name) + execution_id = job_utils.generate_session_id(task_id, task_version, role, party_id) + task_node = job_parser.get_task_node(role=role, party_id=party_id, task_name=task_name) + task_parser = job_parser.task_parser( + task_node=task_node, job_id=job_id, task_name=task_name, role=role, party_id=party_id, + task_id=task_id, execution_id=execution_id, task_version=task_version, + parties=job_parser.get_task_runtime_parties(task_name), + model_id=dag_schema.dag.conf.model_id, model_version=dag_schema.dag.conf.model_version + ) + if is_scheduler: + task = ScheduleTask() + task.f_job_id = job_id + task.f_role = role + task.f_party_id = party_id + task.f_task_name = task_name + task.f_component = task_parser.component_ref + task.f_task_id = task_id + task.f_task_version = task_version + task.f_status = TaskStatus.WAITING + task.f_parties = [party.dict() for party in dag_schema.dag.parties] + ScheduleJobSaver.create_task(task.to_human_model_dict()) + else: + task_parameters = task_parser.task_parameters + if task_parser.engine_run: + task_run.update(task_parser.engine_run) + task_parameters.engine_run = task_run + task_parameters.computing_partitions = dag_schema.dag.conf.computing_partitions + schedule_logger(job_id).info(f"task {task_name} role {role} part id {party_id} task_parameters" + f" {task_parameters.dict()}, provider: {task_parser.provider}") + task = Task() + task.f_job_id = job_id + task.f_role = role + task.f_party_id = party_id + task.f_task_name = task_name + task.f_component = task_parser.component_ref + task.f_task_id = task_id + task.f_task_version = task_version + task.f_scheduler_party_id = dag_schema.dag.conf.scheduler_party_id + task.f_status = TaskStatus.WAITING + task.f_party_status = TaskStatus.WAITING + task.f_execution_id = execution_id + task.f_provider_name = task_parser.provider + task.f_timeout = task_parser.timeout + task.f_sync_type = dag_schema.dag.conf.sync_type + task.f_task_run = task_run + task.f_task_cores = task_cores + cls.update_local(task) + cls.update_launcher_config(task, task_parser.task_runtime_launcher, task_parameters) + task.f_component_parameters = task_parameters.dict() + status = JobSaver.create_task(task.to_human_model_dict()) + schedule_logger(job_id).info(task.to_human_model_dict()) + schedule_logger(job_id).info(status) + + @staticmethod + def update_local(task): + # HA need route to local + if task.f_role == "local": + task.f_run_ip = RuntimeConfig.JOB_SERVER_HOST + task.f_run_port = RuntimeConfig.HTTP_PORT + + @staticmethod + def update_launcher_config(task, launcher_name, task_parameters): + # support deepspeed and other launcher + if task.f_role == "arbiter": + return + schedule_logger(task.f_job_id).info(f"task runtime launcher name: {launcher_name}") + if launcher_name and launcher_name != LauncherType.DEFAULT: + task_parameters.launcher_name = task.f_launcher_name = launcher_name + + @staticmethod + def create_schedule_tasks(job: ScheduleJob, dag_schema): + for party in job.f_parties: + role = party.get("role") + party_ids = party.get("party_id") + for party_id in party_ids: + TaskController.create_tasks(job.f_job_id, role, party_id, dag_schema, is_scheduler=True) + TaskController.create_scheduler_tasks_status(job.f_job_id, dag_schema) + + @classmethod + def create_scheduler_tasks_status(cls, job_id, dag_schema, task_version=0, auto_retries=None, task_name=None): + schedule_logger(job_id).info("start create schedule task status info") + job_parser = JobParser(dag_schema) + if task_name: + task_list = [task_name] + else: + task_list = job_parser.global_topological_sort() + for _task_name in task_list: + task_info = { + "job_id": job_id, + "task_name": _task_name, + "task_id": job_utils.generate_task_id(job_id=job_id, component_name=_task_name), + "task_version": task_version, + "status": TaskStatus.WAITING, + "auto_retries": dag_schema.dag.conf.auto_retries if auto_retries is None else auto_retries, + "sync_type": dag_schema.dag.conf.sync_type + } + ScheduleJobSaver.create_task_scheduler_status(task_info) + schedule_logger(job_id).info("create schedule task status success") + + @classmethod + def start_task(cls, task: Task): + job_id = task.f_job_id + role = task.f_role + party_id = task.f_party_id + task_id = task.f_task_id + task_version = task.f_task_version + schedule_logger(job_id).info( + f"try to start task {task_id} {task_version} on {role} {party_id} executor subprocess") + task_executor_process_start_status = False + task_info = { + "job_id": job_id, + "task_id": task_id, + "task_version": task_version, + "role": role, + "party_id": party_id, + } + is_failed = False + try: + run_parameters = task.f_component_parameters + schedule_logger(job_id).info(f"task run parameters: {run_parameters}") + task_executor_process_start_status = False + + config_dir = job_utils.get_task_directory( + job_id, role, party_id, task.f_task_name, task.f_task_version, input=True + ) + os.makedirs(config_dir, exist_ok=True) + run_parameters_path = os.path.join(config_dir, 'preprocess_parameters.yaml') + with open(run_parameters_path, 'w') as fw: + yaml.dump(run_parameters, fw) + backend_engine = cls.build_task_engine(task.f_provider_name, task.f_launcher_name) + run_info = backend_engine.run( + task=task, + run_parameters=run_parameters, + run_parameters_path=run_parameters_path, + config_dir=config_dir, + log_dir=job_utils.get_job_log_directory(job_id, role, party_id, task.f_task_name), + cwd_dir=job_utils.get_job_directory(job_id, role, party_id, task.f_task_name) + ) + task_info.update(run_info) + task_info["start_time"] = current_timestamp() + task_executor_process_start_status = True + except Exception as e: + schedule_logger(job_id).exception(e) + is_failed = True + finally: + try: + cls.update_task(task_info=task_info) + task_info["party_status"] = TaskStatus.RUNNING + cls.update_task_status(task_info=task_info) + if is_failed: + task_info["party_status"] = TaskStatus.FAILED + cls.update_task_status(task_info=task_info) + except Exception as e: + schedule_logger(job_id).exception(e) + schedule_logger(job_id).info( + "task {} {} on {} {} executor subprocess start {}".format( + task_id, task_version, role, party_id, "success" if task_executor_process_start_status else "failed" + )) + return not is_failed + + @classmethod + def create_new_version_task(cls, task: Task, new_version): + jobs = JobSaver.query_job(job_id=task.f_job_id, role=task.f_role, party_id=task.f_party_id) + if not jobs: + return False + dag_schema = DAGSchema(**jobs[0].f_dag) + job_parser = JobParser(dag_schema) + cls.create_task( + task.f_job_id, task.f_role, task.f_party_id, task.f_task_name, dag_schema, job_parser, + task_run=task.f_task_run, task_cores=task.f_task_cores, is_scheduler=False, task_version=new_version + ) + + @classmethod + def create_new_version_schedule_task(cls, job, task, auto): + # stop old version task + FederatedScheduler.stop_task(task_id=task.f_task_id, command_body={"status": task.f_status}) + # create new version task + if auto: + task.f_auto_retries = task.f_auto_retries - 1 + retry_time = 3 + new_version = task.f_task_version + while True: + if retry_time == 0: + raise Exception(f"create {task.f_task_id} new version {new_version} failed") + new_version = new_version + 1 + status_code, response = FederatedScheduler.rerun_task(task_id=task.f_task_id, task_version=new_version) + if status_code == FederatedSchedulingStatusCode.SUCCESS: + schedule_logger(job_id=job.f_job_id).info( + f"create {task.f_task_id} new version {new_version} success" + ) + break + retry_time -= 1 + dag_schema = DAGSchema(**job.f_dag) + job_parser = JobParser(dag_schema) + parties = job_parser.get_task_runtime_parties(task_name=task.f_task_name) + for party in parties: + for party_id in party.party_id: + cls.create_task( + job.f_job_id, party.role, party_id, task.f_task_name, dag_schema, job_parser, + is_scheduler=True, task_version=new_version + ) + TaskController.create_scheduler_tasks_status( + job.f_job_id, + dag_schema, + task_version=new_version, + auto_retries=task.f_auto_retries, + task_name=task.f_task_name + ) + schedule_logger(job.f_job_id).info(f"create task {task.f_task_id} new version {new_version} successfully") + + @classmethod + def prepare_rerun_task(cls, job: ScheduleJob, task: ScheduleTaskStatus, auto=False, force=False): + job_id = job.f_job_id + can_rerun = False + if force: + can_rerun = True + auto = False + schedule_logger(job_id).info( + f"task {task.f_task_id} {task.f_task_version} with {task.f_status} was forced to rerun") + elif task.f_status in {TaskStatus.SUCCESS}: + schedule_logger(job_id).info( + f"task {task.f_task_id} {task.f_task_version} is {task.f_status} and not force reruen, pass rerun") + elif auto and task.f_auto_retries < 1: + schedule_logger(job_id).info(f"task {task.f_task_id} has no retry count, pass rerun") + else: + can_rerun = True + if can_rerun: + if task.f_status != TaskStatus.WAITING: + cls.create_new_version_schedule_task(job=job, task=task, auto=auto) + return can_rerun + + @classmethod + def update_task(cls, task_info): + update_status = False + try: + update_status = JobSaver.update_task(task_info=task_info) + except Exception as e: + schedule_logger(task_info["job_id"]).exception(e) + finally: + return update_status + + @classmethod + def update_task_status(cls, task_info, scheduler_party_id=None, sync_type=None): + task = JobSaver.query_task( + task_id=task_info.get("task_id"), + task_version=task_info.get("task_version"), + role=task_info.get("role"), + party_id=task_info.get("party_id") + )[0] + scheduler_party_id, sync_type = task.f_scheduler_party_id, task.f_sync_type + update_status = JobSaver.update_task_status(task_info=task_info) + if update_status and EndStatus.contains(task_info.get("party_status")): + ResourceManager.return_task_resource(**task_info) + if "party_status" in task_info: + report_task_info = { + "job_id": task_info.get("job_id"), + "role": task_info.get("role"), + "party_id": task_info.get("party_id"), + "task_id": task_info.get("task_id"), + "task_version": task_info.get("task_version"), + "status": task_info.get("party_status") + } + if sync_type == FederatedCommunicationType.CALLBACK: + cls.report_task_to_scheduler(task_info=report_task_info, scheduler_party_id=scheduler_party_id) + if update_status and EndStatus.contains(task_info.get("party_status")): + cls.callback_task_output(task) + return update_status + + @classmethod + def report_task_to_scheduler(cls, task_info, scheduler_party_id): + FederatedScheduler.report_task_to_scheduler(party_id=scheduler_party_id, command_body=task_info) + + @classmethod + def collect_task(cls, job_id, task_id, task_version, role, party_id): + tasks = JobSaver.query_task(job_id=job_id, task_id=task_id, task_version=task_version, role=role, + party_id=party_id) + if tasks: + return tasks[0].to_human_model_dict(only_primary_with=cls.INITIATOR_COLLECT_FIELDS) + else: + return None + + @classmethod + @asynchronous_function + def stop_task(cls, task: Task, stop_status): + kill_status = cls.kill_task(task=task) + task_info = { + "job_id": task.f_job_id, + "task_id": task.f_task_id, + "task_version": task.f_task_version, + "role": task.f_role, + "party_id": task.f_party_id, + "party_status": stop_status, + "kill_status": True + } + cls.update_task_status(task_info=task_info, scheduler_party_id=task.f_scheduler_party_id, sync_type=task.f_sync_type) + cls.update_task(task_info=task_info) + return kill_status + + @classmethod + def kill_task(cls, task: Task): + kill_status = False + try: + backend_engine = cls.build_task_engine(task.f_provider_name, task.f_launcher_name) + if backend_engine: + backend_engine.kill(task) + backend_engine.cleanup(task) + WorkerManager.kill_task_all_workers(task) + except Exception as e: + schedule_logger(task.f_job_id).exception(e) + else: + kill_status = True + finally: + schedule_logger(task.f_job_id).info( + 'task {} {} on {} {} process {} kill {}'.format( + task.f_task_id, + task.f_task_version, + task.f_role, + task.f_party_id, + task.f_run_pid, + 'success' if kill_status else 'failed' + )) + return kill_status + + @classmethod + def clean_task(cls, task): + try: + backend_engine = cls.build_task_engine(task.f_provider_name, task.f_launcher_name) + if backend_engine: + schedule_logger(task.f_job_id).info(f"start clean task:[{task.f_task_id} {task.f_task_version}]") + backend_engine.cleanup(task) + WorkerManager.kill_task_all_workers(task) + except Exception as e: + schedule_logger(task.f_job_id).exception(e) + + @classmethod + def build_task_engine(cls, provider_name, launcher_name=LauncherType.DEFAULT) -> EngineABC: + return build_engine(provider_name, launcher_name) + + @classmethod + def callback_task_output(cls, task: Task): + if task.f_launcher_name == LauncherType.DEEPSPEED: + engine = cls.build_task_engine(provider_name=task.f_provider_name, launcher_name=task.f_launcher_name) + engine.download_output(task) diff --git a/python/fate_flow/controller/task_controller.py b/python/fate_flow/controller/task_controller.py deleted file mode 100644 index b0f4a8e77..000000000 --- a/python/fate_flow/controller/task_controller.py +++ /dev/null @@ -1,284 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os -import sys - -from fate_arch.common import FederatedCommunicationType -from fate_flow.utils.job_utils import asynchronous_function -from fate_flow.utils.log_utils import schedule_logger -from fate_flow.controller.engine_adapt import build_engine -from fate_flow.db.db_models import Task -from fate_flow.scheduler.federated_scheduler import FederatedScheduler -from fate_flow.entity.run_status import TaskStatus, EndStatus -from fate_flow.utils import job_utils, process_utils -from fate_flow.operation.job_saver import JobSaver -from fate_arch.common.base_utils import json_dumps, current_timestamp -from fate_arch.common import base_utils -from fate_flow.entity import RunParameters -from fate_flow.manager.resource_manager import ResourceManager -from fate_flow.operation.job_tracker import Tracker -from fate_flow.manager.worker_manager import WorkerManager -from fate_flow.entity.types import TaskCleanResourceType, TaskLauncher -from fate_flow.worker.download_model import DownloadModel - - -class TaskController(object): - INITIATOR_COLLECT_FIELDS = ["status", "party_status", "start_time", "update_time", "end_time", "elapsed"] - - @classmethod - def create_task(cls, role, party_id, run_on_this_party, task_info): - task_info["role"] = role - task_info["party_id"] = str(party_id) - task_info["status"] = TaskStatus.WAITING - task_info["party_status"] = TaskStatus.WAITING - task_info["create_time"] = base_utils.current_timestamp() - task_info["run_on_this_party"] = run_on_this_party - if task_info.get("task_id") is None: - task_info["task_id"] = job_utils.generate_task_id(job_id=task_info["job_id"], component_name=task_info["component_name"]) - if task_info.get("task_version") is None: - task_info["task_version"] = 0 - run_parameters_dict = job_utils.get_job_parameters(task_info.get("job_id"), role, party_id) - run_parameters = RunParameters(**run_parameters_dict) - task_info.update({"is_deepspeed": cls.is_deepspeed(run_parameters, role, party_id, task_info["component_name"])}) - task = JobSaver.create_task(task_info=task_info) - - @classmethod - def start_task(cls, job_id, component_name, task_id, task_version, role, party_id, **kwargs): - """ - Start task, update status and party status - :param job_id: - :param component_name: - :param task_id: - :param task_version: - :param role: - :param party_id: - :return: - """ - schedule_logger(job_id).info( - f"try to start task {task_id} {task_version} on {role} {party_id} executor subprocess") - task_executor_process_start_status = False - task_info = { - "job_id": job_id, - "task_id": task_id, - "task_version": task_version, - "role": role, - "party_id": party_id, - } - is_failed = False - try: - task = JobSaver.query_task(task_id=task_id, task_version=task_version, role=role, party_id=party_id)[0] - run_parameters_dict = job_utils.get_job_parameters(job_id, role, party_id) - run_parameters_dict["src_user"] = kwargs.get("src_user") - run_parameters = RunParameters(**run_parameters_dict) - - config_dir = job_utils.get_task_directory(job_id, role, party_id, component_name, task_id, task_version) - os.makedirs(config_dir, exist_ok=True) - - run_parameters_path = os.path.join(config_dir, 'task_parameters.json') - with open(run_parameters_path, 'w') as fw: - fw.write(json_dumps(run_parameters_dict)) - - schedule_logger(job_id).info(f"use computing engine {run_parameters.computing_engine}") - task_info["engine_conf"] = {"computing_engine": run_parameters.computing_engine} - backend_engine = build_engine( - run_parameters.computing_engine, - task.f_is_deepspeed) - run_info = backend_engine.run(task=task, - run_parameters=run_parameters, - run_parameters_path=run_parameters_path, - config_dir=config_dir, - log_dir=job_utils.get_job_log_directory(job_id, role, party_id, component_name), - cwd_dir=job_utils.get_job_directory(job_id, role, party_id, component_name), - user_name=kwargs.get("user_id")) - task_info.update(run_info) - task_info["start_time"] = current_timestamp() - task_executor_process_start_status = True - except Exception as e: - schedule_logger(job_id).exception(e) - is_failed = True - finally: - try: - cls.update_task(task_info=task_info) - task_info["party_status"] = TaskStatus.RUNNING - cls.update_task_status(task_info=task_info) - if is_failed: - task_info["party_status"] = TaskStatus.FAILED - cls.update_task_status(task_info=task_info) - except Exception as e: - schedule_logger(job_id).exception(e) - schedule_logger(job_id).info( - "task {} {} on {} {} executor subprocess start {}".format(task_id, task_version, role, party_id, "success" if task_executor_process_start_status else "failed")) - - @classmethod - def update_task(cls, task_info): - """ - Save to local database and then report to Initiator - :param task_info: - :return: - """ - update_status = False - try: - update_status = JobSaver.update_task(task_info=task_info) - cls.report_task_to_initiator(task_info=task_info) - except Exception as e: - schedule_logger(task_info["job_id"]).exception(e) - finally: - return update_status - - @classmethod - def update_task_status(cls, task_info): - update_status = JobSaver.update_task_status(task_info=task_info) - task = JobSaver.query_task(task_id=task_info["task_id"], - task_version=task_info["task_version"], - role=task_info["role"], - party_id=task_info["party_id"])[0] - if update_status and EndStatus.contains(task_info.get("status")): - ResourceManager.return_task_resource(task_info=task_info) - cls.clean_task(job_id=task_info["job_id"], - task_id=task_info["task_id"], - task_version=task_info["task_version"], - role=task_info["role"], - party_id=task_info["party_id"], - content_type=TaskCleanResourceType.TABLE, - is_asynchronous=True) - cls.report_task_to_initiator(task_info=task_info, task=task) - cls.callback_task_output(task, task_info.get("status")) - return update_status - - @classmethod - def report_task_to_initiator(cls, task_info, task=None): - if not task: - tasks = JobSaver.query_task(task_id=task_info["task_id"], - task_version=task_info["task_version"], - role=task_info["role"], - party_id=task_info["party_id"]) - task = tasks[0] - if task_info.get("error_report"): - task.f_error_report = task_info.get("error_report") - if task.f_federated_status_collect_type == FederatedCommunicationType.PUSH: - FederatedScheduler.report_task_to_initiator(task=task) - - @classmethod - def collect_task(cls, job_id, component_name, task_id, task_version, role, party_id): - tasks = JobSaver.query_task(job_id=job_id, component_name=component_name, task_id=task_id, task_version=task_version, role=role, party_id=party_id) - if tasks: - return tasks[0].to_human_model_dict(only_primary_with=cls.INITIATOR_COLLECT_FIELDS) - else: - return None - - @classmethod - @asynchronous_function - def stop_task(cls, task, stop_status): - """ - Try to stop the task, but the status depends on the final operation result - :param task: - :param stop_status: - :return: - """ - kill_status = cls.kill_task(task=task) - task_info = { - "job_id": task.f_job_id, - "task_id": task.f_task_id, - "task_version": task.f_task_version, - "role": task.f_role, - "party_id": task.f_party_id, - "party_status": stop_status, - "kill_status": True - } - cls.update_task_status(task_info=task_info) - cls.update_task(task_info=task_info) - return kill_status - - @classmethod - def kill_task(cls, task: Task): - kill_status = False - try: - # kill task executor - try: - backend_engine = build_engine( - task.f_engine_conf.get("computing_engine"), - task.f_is_deepspeed - ) - if backend_engine: - backend_engine.kill(task) - except Exception as e: - schedule_logger(task.f_job_id).exception(e) - WorkerManager.kill_task_all_workers(task) - except Exception as e: - schedule_logger(task.f_job_id).exception(e) - else: - kill_status = True - finally: - schedule_logger(task.f_job_id).info( - 'task {} {} on {} {} process {} kill {}'.format(task.f_task_id, - task.f_task_version, - task.f_role, - task.f_party_id, - task.f_run_pid, - 'success' if kill_status else 'failed')) - return kill_status - - @classmethod - @asynchronous_function - def clean_task(cls, job_id, task_id, task_version, role, party_id, content_type: TaskCleanResourceType): - status = set() - if content_type == TaskCleanResourceType.METRICS: - tracker = Tracker(job_id=job_id, role=role, party_id=party_id, task_id=task_id, task_version=task_version) - status.add(tracker.clean_metrics()) - elif content_type == TaskCleanResourceType.TABLE: - jobs = JobSaver.query_job(job_id=job_id, role=role, party_id=party_id) - if jobs: - job = jobs[0] - job_parameters = RunParameters(**job.f_runtime_conf_on_party["job_parameters"]) - tracker = Tracker(job_id=job_id, role=role, party_id=party_id, task_id=task_id, task_version=task_version, job_parameters=job_parameters) - status.add(tracker.clean_task()) - if len(status) == 1 and True in status: - return True - else: - return False - - @staticmethod - def is_deepspeed(run_parameters, role, party_id, component_name): - task_conf = run_parameters.role_parameter("task_conf", role=role, party_id=party_id) - if task_conf.get(component_name, {}).get("launcher") == TaskLauncher.DEEPSPEED.value and role != "arbiter": - return True - else: - return False - - @staticmethod - def callback_task_output(task, status): - if EndStatus.contains(status): - if task.f_is_deepspeed: - deepspeed_engine = build_engine(task.f_engine_conf.get("computing_engine"), task.f_is_deepspeed) - deepspeed_engine.download_log(task) - - # run subprocess to download model - conf_dir = job_utils.get_job_directory(job_id=task.f_job_id) - os.makedirs(conf_dir, exist_ok=True) - process_cmd = [ - sys.executable or 'python3', - sys.modules[DownloadModel.__module__].__file__, - '--job_id', task.f_job_id, - '--role', task.f_role, - '--party_id', task.f_party_id, - '--task_id', task.f_task_id, - '--task_version', task.f_task_version, - '--computing_engine', task.f_engine_conf.get("computing_engine") - ] - process_name = "model_download" - log_dir = job_utils.get_job_log_directory(job_id=task.f_job_id) - process_utils.run_subprocess(job_id=task.f_job_id, config_dir=conf_dir, process_cmd=process_cmd, - log_dir=log_dir, process_name=process_name) diff --git a/python/fate_flow/controller/tests/job_controller_test.py b/python/fate_flow/controller/tests/job_controller_test.py deleted file mode 100644 index 47f59c0b6..000000000 --- a/python/fate_flow/controller/tests/job_controller_test.py +++ /dev/null @@ -1,100 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import unittest - -from fate_flow.utils.base_utils import jprint -from fate_flow.controller.job_controller import JobController -from fate_flow.utils import job_utils - - -class TestJobController(unittest.TestCase): - def test_gen_updated_parameters(self): - job_id = "202110211127411105150" - initiator_role = "guest" - initiator_party_id = 9999 - input_job_parameters = { - "common": { - "auto_retries": 1, - "auto_retry_delay": 1 - } - } - input_component_parameters = { - "common": { - "hetero_lr_0": { - "alpha": 0.02 - } - }, - "role": { - "guest": { - "0": { - "reader_0": { - "table": {"name": "breast_hetero_guest", "namespace": "unitest_experiment"} - }, - "homo_nn_0":{ - "with_label": True, - "output_format": "dense" - }, - } - }, - "host": { - "1": { - "dataio_0":{ - "with_label": True, - "output_format": "dense" - }, - "evaluation_0": { - "need_run": True - } - } - } - } - } - - updated_job_parameters, updated_component_parameters, updated_components = JobController.gen_updated_parameters( - job_id=job_id, - initiator_role=initiator_role, - initiator_party_id=initiator_party_id, - input_job_parameters=input_job_parameters, - input_component_parameters=input_component_parameters) - jprint(updated_job_parameters) - jprint(updated_component_parameters) - self.assertTrue(check(input_component_parameters, updated_component_parameters)[0]) - # todo: add check with origin parameters and add dsl parser check - - -def check(inputs, result): - # todo: return check keys chain - if type(result) != type(inputs): - return False, "type not match" - elif isinstance(inputs, dict): - for k, v in inputs.items(): - if k not in result: - return False, f"no such {k} key" - if isinstance(v, (dict, list)): - return check(v, result[k]) - else: - if result[k] != v: - return False, f"{k} value not match" - else: - return True, "match" - elif isinstance(inputs, list): - return result == inputs - else: - raise Exception(f"not support type {type(inputs)}") - - -if __name__ == '__main__': - unittest.main() diff --git a/python/fate_flow/controller/version_controller.py b/python/fate_flow/controller/version_controller.py deleted file mode 100644 index 1a3f1a8d6..000000000 --- a/python/fate_flow/controller/version_controller.py +++ /dev/null @@ -1,116 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# algorithm version compatibility control -from fate_arch.common import file_utils -from fate_flow.settings import INCOMPATIBLE_VERSION_CONF - - -class VersionController: - INCOMPATIBLE_VERSION = {} - - @classmethod - def init(cls): - try: - conf = file_utils.load_yaml_conf(INCOMPATIBLE_VERSION_CONF) - for key, key_version in conf.items(): - cls.INCOMPATIBLE_VERSION[key] = {} - for version in conf[key]: - cls.INCOMPATIBLE_VERSION[key][str(version)] = conf[key][version] - except Exception as e: - pass - - @classmethod - def job_provider_version_check(cls, providers_info, local_role, local_party_id): - incompatible_info = {} - incompatible = False - if local_role in providers_info: - local_provider = providers_info[local_role].get(int(local_party_id), {}) \ - or providers_info[local_role].get(str(local_party_id), {}) - for role, role_provider in providers_info.items(): - incompatible_info[role] = {} - for party_id, provider in role_provider.items(): - if role == local_role and str(party_id) == str(local_party_id): - continue - role_incompatible_info = cls.provider_version_check(local_provider, party_provider=provider) - if role_incompatible_info: - incompatible = True - incompatible_info[role][party_id] = role_incompatible_info - if incompatible: - raise ValueError(f"version compatibility check failed: {incompatible_info}") - - @classmethod - def provider_version_check(cls, local_provider, party_provider): - incompatible_info = {} - for component, info in local_provider.items(): - if party_provider.get(component): - local_version = local_provider.get(component).get("provider").get("version") - party_version = party_provider.get(component).get("provider").get("version") - if cls.is_incompatible(local_version, party_version): - if component in incompatible_info: - incompatible_info[component].append((local_version, party_version)) - else: - incompatible_info[component] = [(local_version, party_version)] - return incompatible_info - - @classmethod - def is_incompatible(cls, source_version, dest_version, key="FATE"): - if not source_version or not dest_version: - return False - index = len(source_version) - while True: - if source_version[:index] in cls.INCOMPATIBLE_VERSION.get(key, {}).keys(): - for incompatible_value in cls.INCOMPATIBLE_VERSION.get(key)[source_version[:index]].split(","): - if cls.is_match(dest_version, incompatible_value.strip()): - return True - index -= 1 - if index == 0: - return False - - @classmethod - def is_match(cls, dest_ver, incompatible_value): - symbols, incompatible_ver = cls.extract_symbols(incompatible_value) - dest_ver_list = cls.extend_version([int(_) for _ in dest_ver.split(".")]) - incompatible_ver_list = cls.extend_version([int(_) for _ in incompatible_ver.split(".")]) - print(dest_ver_list, incompatible_ver_list, symbols) - for index in range(4): - if dest_ver_list[index] == incompatible_ver_list[index]: - continue - if dest_ver_list[index] > incompatible_ver_list[index]: - return True if ">" in symbols else False - if dest_ver_list[index] < incompatible_ver_list[index]: - return True if "<" in symbols else False - return True if "=" in symbols else False - - @classmethod - def extend_version(cls, v): - v_len = len(v) - if v_len < 4: - for i in range(4 - v_len): - v.append(0) - return v - - @classmethod - def extract_symbols(cls, incompatible_value): - symbols_list = ["<", ">", "="] - index = 0 - for index, ver in enumerate(incompatible_value): - if ver not in symbols_list: - break - symbol = incompatible_value[0: index] - if not incompatible_value[0: index]: - symbol = "=" - return symbol, incompatible_value[index:] diff --git a/python/fate_flow/db/__init__.py b/python/fate_flow/db/__init__.py index 878d3a9c5..1556878ce 100644 --- a/python/fate_flow/db/__init__.py +++ b/python/fate_flow/db/__init__.py @@ -13,3 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from fate_flow.db.storage_models import * +from fate_flow.db.schedule_models import * +from fate_flow.db.db_models import * diff --git a/python/fate_flow/db/base_models.py b/python/fate_flow/db/base_models.py new file mode 100644 index 000000000..0ddf0442d --- /dev/null +++ b/python/fate_flow/db/base_models.py @@ -0,0 +1,461 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import operator +import os +from enum import IntEnum +from functools import wraps + +import typing + +import peewee +from peewee import ( + BigIntegerField, + CompositeKey, + Field, + FloatField, + IntegerField, + Metadata, + Model, + TextField +) +from playhouse.pool import PooledMySQLDatabase + +from fate_flow.hub.flow_hub import FlowHub + +from fate_flow.runtime.system_settings import DATABASE +from fate_flow.utils.base_utils import json_dumps, json_loads, date_string_to_timestamp, \ + current_timestamp, timestamp_to_date +from fate_flow.utils.log_utils import getLogger, sql_logger +from fate_flow.utils.object_utils import from_dict_hook + +CONTINUOUS_FIELD_TYPE = {IntegerField, FloatField} +AUTO_DATE_TIMESTAMP_FIELD_PREFIX = { + "create", + "start", + "end", + "update", + "read_access", + "write_access", +} + + +LOGGER = getLogger() + + +class SerializedType(IntEnum): + PICKLE = 1 + JSON = 2 + + +class LongTextField(TextField): + field_type = "LONGTEXT" + + +class JSONField(LongTextField): + default_value = {} + + def __init__(self, object_hook=None, object_pairs_hook=None, **kwargs): + self._object_hook = object_hook + self._object_pairs_hook = object_pairs_hook + super().__init__(**kwargs) + + def db_value(self, value): + if value is None: + value = self.default_value + return json_dumps(value) + + def python_value(self, value): + if not value: + return self.default_value + return json_loads( + value, + object_hook=self._object_hook, + object_pairs_hook=self._object_pairs_hook, + ) + + +class ListField(JSONField): + default_value = [] + + +class SerializedField(LongTextField): + def __init__( + self, + serialized_type=SerializedType.PICKLE, + object_hook=None, + object_pairs_hook=None, + **kwargs, + ): + self._serialized_type = serialized_type + self._object_hook = object_hook + self._object_pairs_hook = object_pairs_hook + super().__init__(**kwargs) + + def db_value(self, value): + if self._serialized_type == SerializedType.JSON: + if value is None: + return None + return json_dumps(value, with_type=True) + else: + raise ValueError( + f"the serialized type {self._serialized_type} is not supported" + ) + + def python_value(self, value): + if self._serialized_type == SerializedType.JSON: + if value is None: + return {} + return json_loads( + value, + object_hook=self._object_hook, + object_pairs_hook=self._object_pairs_hook, + ) + else: + raise ValueError( + f"the serialized type {self._serialized_type} is not supported" + ) + + +def is_continuous_field(cls: typing.Type) -> bool: + if cls in CONTINUOUS_FIELD_TYPE: + return True + for p in cls.__bases__: + if p in CONTINUOUS_FIELD_TYPE: + return True + elif p != Field and p != object: + if is_continuous_field(p): + return True + else: + return False + + +class JsonSerializedField(SerializedField): + def __init__(self, object_hook=from_dict_hook, object_pairs_hook=None, **kwargs): + super(JsonSerializedField, self).__init__(serialized_type=SerializedType.JSON, object_hook=object_hook, + object_pairs_hook=object_pairs_hook, **kwargs) + + +def singleton(cls, *args, **kw): + instances = {} + + def _singleton(): + key = str(cls) + str(os.getpid()) + if key not in instances: + instances[key] = cls(*args, **kw) + return instances[key] + + return _singleton + + +def auto_date_timestamp_field(): + return {f"{f}_time" for f in AUTO_DATE_TIMESTAMP_FIELD_PREFIX} + + +def auto_date_timestamp_db_field(): + return {f"f_{f}_time" for f in AUTO_DATE_TIMESTAMP_FIELD_PREFIX} + + +def remove_field_name_prefix(field_name): + return field_name[2:] if field_name.startswith("f_") else field_name + + +@singleton +class BaseDataBase: + def __init__(self): + engine_name = DATABASE.get("engine") + config = DATABASE.get(engine_name) + decrypt_key = DATABASE.get("decrypt_key") + self.database_connection = FlowHub.load_database(engine_name, config, decrypt_key) + + +class DatabaseLock: + def __init__(self, lock_name, timeout=10, db=None): + self.lock_name = lock_name + self.timeout = int(timeout) + self.db = db if db else DB + + def lock(self): + # SQL parameters only support %s format placeholders + cursor = self.db.execute_sql("SELECT GET_LOCK(%s, %s)", (self.lock_name, self.timeout)) + ret = cursor.fetchone() + if ret[0] == 0: + raise Exception(f'acquire mysql lock {self.lock_name} timeout') + elif ret[0] == 1: + return True + else: + raise Exception(f'failed to acquire lock {self.lock_name}') + + def unlock(self): + cursor = self.db.execute_sql("SELECT RELEASE_LOCK(%s)", (self.lock_name, )) + ret = cursor.fetchone() + if ret[0] == 0: + raise Exception(f'mysql lock {self.lock_name} was not established by this thread') + elif ret[0] == 1: + return True + else: + raise Exception(f'mysql lock {self.lock_name} does not exist') + + def __enter__(self): + if isinstance(self.db, PooledMySQLDatabase): + self.lock() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if isinstance(self.db, PooledMySQLDatabase): + self.unlock() + + def __call__(self, func): + @wraps(func) + def magic(*args, **kwargs): + with self: + return func(*args, **kwargs) + return magic + + +DB = BaseDataBase().database_connection +DB.lock = DatabaseLock + + +def close_connection(): + try: + if DB: + DB.close() + except Exception as e: + LOGGER.exception(e) + + +class BaseModel(Model): + f_create_time = BigIntegerField(null=True) + f_update_time = BigIntegerField(null=True) + + def to_json(self): + # This function is obsolete + return self.to_dict() + + def to_dict(self): + return self.__dict__["__data__"] + + def to_human_model_dict(self, only_primary_with: list = None): + model_dict = self.__dict__["__data__"] + + if not only_primary_with: + return {remove_field_name_prefix(k): v for k, v in model_dict.items()} + + human_model_dict = {} + for k in self._meta.primary_key.field_names: + human_model_dict[remove_field_name_prefix(k)] = model_dict[k] + for k in only_primary_with: + human_model_dict[k] = model_dict[f"f_{k}"] + return human_model_dict + + @property + def meta(self) -> Metadata: + return self._meta + + @classmethod + def get_primary_keys_name(cls): + return ( + cls._meta.primary_key.field_names + if isinstance(cls._meta.primary_key, CompositeKey) + else [cls._meta.primary_key.name] + ) + + @classmethod + def getter_by(cls, attr): + return operator.attrgetter(attr)(cls) + + @classmethod + def query(cls, reverse=None, order_by=None, force=False, **kwargs): + filters = [] + for f_n, f_v in kwargs.items(): + attr_name = "f_%s" % f_n + if not hasattr(cls, attr_name) or f_v is None: + continue + if type(f_v) in {list, set}: + f_v = list(f_v) + if is_continuous_field(type(getattr(cls, attr_name))): + if len(f_v) == 2: + for i, v in enumerate(f_v): + if ( + isinstance(v, str) + and f_n in auto_date_timestamp_field() + ): + # time type: %Y-%m-%d %H:%M:%S + f_v[i] = date_string_to_timestamp(v) + lt_value = f_v[0] + gt_value = f_v[1] + if lt_value is not None and gt_value is not None: + filters.append( + cls.getter_by(attr_name).between(lt_value, gt_value) + ) + elif lt_value is not None: + filters.append( + operator.attrgetter(attr_name)(cls) >= lt_value + ) + elif gt_value is not None: + filters.append( + operator.attrgetter(attr_name)(cls) <= gt_value + ) + else: + filters.append(operator.attrgetter(attr_name)(cls) << f_v) + else: + filters.append(operator.attrgetter(attr_name)(cls) == f_v) + if filters: + query_records = cls.select().where(*filters) + if reverse is not None: + if isinstance(order_by, str) or not order_by: + if not order_by or not hasattr(cls, f"f_{order_by}"): + order_by = "create_time" + query_records = cls.desc(query_records=query_records, reverse=[reverse], order_by=[order_by]) + elif isinstance(order_by, list): + if not isinstance(reverse, list) or len(reverse) != len(order_by): + raise ValueError(f"reverse need is list and length={len(order_by)}") + query_records = cls.desc(query_records=query_records, reverse=reverse, order_by=order_by) + else: + raise ValueError(f"order_by type {type(order_by)} not support") + return [query_record for query_record in query_records] + + elif force: + # force query all + query_records = cls.select() + return [query_record for query_record in query_records] + else: + return [] + + @classmethod + def desc(cls, query_records, order_by: list, reverse: list): + _filters = list() + for _k, _ob in enumerate(order_by): + if reverse[_k] is True: + _filters.append(cls.getter_by(f"f_{_ob}").desc()) + else: + _filters.append(cls.getter_by(f"f_{_ob}").asc()) + return query_records.order_by(*tuple(_filters)) + + @classmethod + def insert(cls, __data=None, **insert): + if isinstance(__data, dict) and __data: + __data[cls._meta.combined["f_create_time"]] = current_timestamp() + if insert: + insert["f_create_time"] = current_timestamp() + + return super().insert(__data, **insert) + + # update and insert will call this method + @classmethod + def _normalize_data(cls, data, kwargs): + normalized = super()._normalize_data(data, kwargs) + if not normalized: + return {} + + normalized[cls._meta.combined["f_update_time"]] = current_timestamp() + + for f_n in AUTO_DATE_TIMESTAMP_FIELD_PREFIX: + if ( + {f"f_{f_n}_time", f"f_{f_n}_date"}.issubset(cls._meta.combined.keys()) + and cls._meta.combined[f"f_{f_n}_time"] in normalized + and normalized[cls._meta.combined[f"f_{f_n}_time"]] is not None + ): + normalized[cls._meta.combined[f"f_{f_n}_date"]] = timestamp_to_date( + normalized[cls._meta.combined[f"f_{f_n}_time"]] + ) + + return normalized + + +class DataBaseModel(BaseModel): + class Meta: + database = DB + + +@DB.connection_context() +def init_database_tables(): + table_objs = [] + create_failed_list = [] + for obj in [name for name in DataBaseModel.__subclasses__()]: + table_objs.append(obj) + LOGGER.info(f"start create table {obj.__name__}") + try: + obj.create_table() + LOGGER.info(f"create table success: {obj.__name__}") + except Exception as e: + LOGGER.exception(e) + create_failed_list.append(obj.__name__) + if create_failed_list: + LOGGER.info(f"create tables failed: {create_failed_list}") + raise Exception(f"create tables failed: {create_failed_list}") + + +def fill_db_model_object(model_object, human_model_dict): + for k, v in human_model_dict.items(): + attr_name = 'f_%s' % k + if hasattr(model_object.__class__, attr_name): + setattr(model_object, attr_name, v) + return model_object + + +class BaseModelOperate: + @classmethod + @DB.connection_context() + def _create_entity(cls, entity_model, entity_info: dict) -> object: + obj = entity_model() + obj.f_create_time = current_timestamp() + for k, v in entity_info.items(): + attr_name = 'f_%s' % k + if hasattr(entity_model, attr_name): + setattr(obj, attr_name, v) + try: + rows = obj.save(force_insert=True) + if rows != 1: + raise Exception("Create {} failed".format(entity_model)) + return obj + except peewee.IntegrityError as e: + # if e.args[0] == 1062 or (isinstance(e.args[0], str) and "UNIQUE constraint failed" in e.args[0]): + # sql_logger(job_id=entity_info.get("job_id", "fate_flow")).warning(e) + # else: + # raise Exception("Create {} failed:\n{}".format(entity_model, e)) + # raise Exception(e) + pass + except Exception as e: + raise Exception("Create {} failed:\n{}".format(entity_model, e)) + + @classmethod + @DB.connection_context() + def _query(cls, entity_model, force=False, **kwargs): + return entity_model.query(force=force, **kwargs) + + @classmethod + @DB.connection_context() + def _delete(cls, entity_model, **kwargs): + _kwargs = {} + filters = [] + for f_k, f_v in kwargs.items(): + attr_name = "f_%s" % f_k + filters.append(operator.attrgetter(attr_name)(entity_model) == f_v) + return entity_model.delete().where(*filters).execute() > 0 + + @classmethod + def safe_save(cls, model, defaults, **kwargs): + entity_model, status = model.get_or_create( + **kwargs, + defaults=defaults) + if status is False: + for key in defaults: + setattr(entity_model, key, defaults[key]) + entity_model.save(force_insert=False) + return "update" + return "create" + diff --git a/python/fate_flow/db/casbin_models.py b/python/fate_flow/db/casbin_models.py new file mode 100644 index 000000000..a5f0a4f2e --- /dev/null +++ b/python/fate_flow/db/casbin_models.py @@ -0,0 +1,252 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from functools import reduce + +import casbin +import peewee as pw + +from fate_flow.db.base_models import singleton, DB +from fate_flow.runtime.system_settings import CASBIN_MODEL_CONF, CASBIN_TABLE_NAME, PERMISSION_TABLE_NAME, \ + PERMISSION_CASBIN_MODEL_CONF + + +class FlowCasbinAdapter(casbin.persist.Adapter): + def __init__(self, rule=None): + if not rule: + rule = FlowCasbinRule + self.rule = rule + self.database = DB + proxy = pw.Proxy() + self.rule._meta.database = proxy + proxy.initialize(DB) + + def load_policy(self, model): + for line in self.rule.select(): + casbin.persist.load_policy_line(str(line), model) + + def _save_policy_line(self, ptype, rule): + data = dict(zip(['v0', 'v1', 'v2', 'v3', 'v4', 'v5'], rule)) + item = self.rule(ptype=ptype) + item.__data__.update(data) + item.save() + + def save_policy(self, model): + """saves all policy rules to the storage.""" + for sec in ["p", "g"]: + if sec not in model.model.keys(): + continue + for ptype, ast in model.model[sec].items(): + for rule in ast.policy: + self._save_policy_line(ptype, rule) + return True + + def add_policy(self, sec, ptype, rule): + """adds a policy rule to the storage.""" + self._save_policy_line(ptype, rule) + + def remove_policy(self, sec, ptype, rule): + """removes a policy rule from the storage.""" + if sec in ["p", "g"]: + condition = [self.rule.ptype == ptype] + data = dict(zip(['v0', 'v1', 'v2', 'v3', 'v4', 'v5'], rule)) + condition.extend([getattr(self.rule, k) == data[k] for k in data]) + check = self.rule.select().filter(*condition) + if check.exists(): + self.rule.delete().where(*condition).execute() + return True + else: + return False + else: + return False + + def remove_filtered_policy(self, sec, ptype, field_index, *field_values): + """removes policy rules that match the filter from the storage. + This is part of the Auto-Save feature. + """ + pass + + +class FlowCasbinRule(pw.Model): + class Meta: + table_name = CASBIN_TABLE_NAME + + ptype = pw.CharField(max_length=255, null=True) + v0 = pw.CharField(max_length=255, null=True) + v1 = pw.CharField(max_length=255, null=True) + v2 = pw.CharField(max_length=255, null=True) + v3 = pw.CharField(max_length=255, null=True) + v4 = pw.CharField(max_length=255, null=True) + v5 = pw.CharField(max_length=255, null=True) + + def __str__(self): + return reduce(lambda x, y: str(x) + ', ' + str(y) if y else x, + [self.ptype, self.v0, self.v1, self.v2, self.v3, self.v4, self.v5]) + + def __repr__(self): + if not self.id: + return "<{cls}: {desc}>".format(cls=self.__class__.__name__, desc=self) + return "<{cls} {pk}: {desc}>".format(cls=self.__class__.__name__, pk=self.id, desc=self) + + +class PermissionCasbinRule(pw.Model): + class Meta: + table_name = PERMISSION_TABLE_NAME + + ptype = pw.CharField(max_length=255, null=True) + v0 = pw.CharField(max_length=255, null=True) + v1 = pw.CharField(max_length=255, null=True) + v2 = pw.CharField(max_length=255, null=True) + v3 = pw.CharField(max_length=255, null=True) + v4 = pw.CharField(max_length=255, null=True) + v5 = pw.CharField(max_length=255, null=True) + + def __str__(self): + return reduce(lambda x, y: str(x) + ', ' + str(y) if y else x, + [self.ptype, self.v0, self.v1, self.v2, self.v3, self.v4, self.v5]) + + def __repr__(self): + if not self.id: + return "<{cls}: {desc}>".format(cls=self.__class__.__name__, desc=self) + return "<{cls} {pk}: {desc}>".format(cls=self.__class__.__name__, pk=self.id, desc=self) + + +class FlowEnforcer(casbin.Enforcer): + @property + def reload_policy(self): + self.load_policy() + return self + + +@singleton +class FateCasbin(object): + def __init__(self): + self.adapter = None + self.init_adapter() + self._e = FlowEnforcer(CASBIN_MODEL_CONF, self.adapter) + + def init_adapter(self): + self.adapter = FlowCasbinAdapter() + self.init_table() + + @staticmethod + def init_table(): + FlowCasbinRule.create_table() + + @property + def re(self) -> casbin.Enforcer: + return self._e.reload_policy + + @property + def e(self) -> casbin.Enforcer: + return self._e + + def add_policy(self, role, resource, permission): + return self.e.add_policy(role, resource, permission) + + def remove_policy(self, role, resource, permission): + return self.e.remove_policy(role, resource, permission) + + def add_role_for_user(self, user, role): + return self.e.add_role_for_user(user, role) + + def delete_role_for_suer(self, user, role): + return self.e.delete_role_for_user(user, role) + + def delete_roles_for_user(self, user): + return self.e.delete_roles_for_user(user) + + def delete_user(self, user): + return self.e.delete_user(user) + + def delete_role(self, role): + return self.e.delete_role(role) + + def delete_permission(self, *permission): + return self.e.delete_permission(*permission) + + def delete_permissions_for_user(self, user): + return self.e.delete_permissions_for_user(user) + + def get_roles_for_user(self, user): + return self.re.get_roles_for_user(user) + + def get_users_for_role(self, role): + return self.re.get_users_for_role(role) + + def has_role_for_user(self, user, role): + return self.re.has_role_for_user(user, role) + + def has_permission_for_user(self, user, *permission): + return self.re.has_permission_for_user(user, *permission) + + def get_permissions_for_user(self, user): + return self.re.get_permissions_for_user(user) + + def enforcer(self, *rvals): + return self.re.enforce(*rvals) + + +@singleton +class PermissionCasbin(object): + def __init__(self): + self.adapter = None + self.init_adapter() + self._e = FlowEnforcer(PERMISSION_CASBIN_MODEL_CONF, self.adapter) + + def init_adapter(self): + self.adapter = FlowCasbinAdapter(rule=PermissionCasbinRule) + self.init_table() + + @staticmethod + def init_table(): + PermissionCasbinRule.create_table() + + @property + def re(self) -> casbin.Enforcer: + return self._e.reload_policy + + @property + def e(self) -> casbin.Enforcer: + return self._e + + def query(self, party_id): + return self.re.get_permissions_for_user(party_id) + + def delete(self, party_id, type, value): + return self.re.delete_permission_for_user(party_id, type, value) + + def delete_all(self, party_id, type): + return self.re.remove_filtered_policy(0, party_id, type) + + def grant(self, party_id, type, value): + return self.re.add_permission_for_user(party_id, type, value) + + def enforce(self, party_id, type, value): + try: + return self.re.enforce(party_id, type, str(value)) + except Exception as e: + raise Exception(f"{party_id}, {type}, {value} {e}") + + +FATE_CASBIN = None +PERMISSION_CASBIN = None + + +def init_casbin(): + global FATE_CASBIN + global PERMISSION_CASBIN + FATE_CASBIN = FateCasbin() + PERMISSION_CASBIN = PermissionCasbin() diff --git a/python/fate_flow/db/component_registry.py b/python/fate_flow/db/component_registry.py deleted file mode 100644 index 0d20064dc..000000000 --- a/python/fate_flow/db/component_registry.py +++ /dev/null @@ -1,199 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_arch.common import file_utils -from fate_arch.common.versions import get_versions - -from fate_flow.component_env_utils import provider_utils -from fate_flow.db.db_models import ComponentProviderInfo, ComponentRegistryInfo, ComponentInfo, DB -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.entity import ComponentProvider -from fate_flow.entity.types import ComponentProviderName -from fate_flow.settings import FATE_FLOW_DEFAULT_COMPONENT_REGISTRY_PATH -from fate_flow.utils.log_utils import getLogger - -LOGGER = getLogger() - - -class ComponentRegistry: - REGISTRY = {} - - @classmethod - def load(cls): - component_registry = cls.get_from_db(file_utils.load_json_conf_real_time(FATE_FLOW_DEFAULT_COMPONENT_REGISTRY_PATH)) - cls.REGISTRY.update(component_registry) - for provider_name, provider_info in cls.REGISTRY.get("providers", {}).items(): - if not ComponentProviderName.valid(provider_name): - raise Exception(f"not support component provider: {provider_name}") - cls.REGISTRY["providers"] = cls.REGISTRY.get("providers", {}) - cls.REGISTRY["components"] = cls.REGISTRY.get("components", {}) - RuntimeConfig.load_component_registry() - - @classmethod - def register_provider(cls, provider: ComponentProvider): - provider_interface = provider_utils.get_provider_interface(provider) - support_components = provider_interface.get_names() - components = {} - for component_alias, info in support_components.items(): - component_name = component_alias.lower() - if component_name not in components: - components[component_name] = info - elif components[component_name].get("module") != info.get("module"): - raise ValueError(f"component {component_name} have different module info") - components[component_name]["alias"] = components[component_name].get("alias", set()) - components[component_name]["alias"].add(component_alias) - register_info = { - "default": { - "version": provider.version - } - } - register_info = cls.get_providers().get(provider.name, register_info) - register_info[provider.version] = { - "path": provider.path, - "class_path": provider.class_path, - "components": components - } - cls.REGISTRY["providers"][provider.name] = register_info - return components - - @classmethod - def register_components(cls, provider_name, components: dict): - for component_name, info in components.items(): - if component_name not in cls.REGISTRY["components"]: - cls.REGISTRY["components"][component_name] = { - "default_provider": provider_name, - "support_provider": [], - "alias": info["alias"] - } - if provider_name not in cls.REGISTRY["components"][component_name]["support_provider"]: - # do not use set because the json format is not supported - cls.REGISTRY["components"][component_name]["support_provider"].append(provider_name) - for component_alias in info["alias"]: - cls.REGISTRY["components"][component_alias] = cls.REGISTRY["components"][component_name] - - @classmethod - def dump(cls): - cls.save_to_db() - - @classmethod - @DB.connection_context() - @DB.lock("component_register") - def save_to_db(cls): - # save component registry info - for provider_name, provider_group_info in cls.REGISTRY["providers"].items(): - for version, version_register_info in provider_group_info.items(): - if version != "default": - version_info = { - "f_path": version_register_info.get("path"), - "f_python": version_register_info.get("python", ""), - "f_class_path": version_register_info.get("class_path"), - "f_version": version, - "f_provider_name": provider_name - } - cls.safe_save(ComponentProviderInfo, version_info, f_version=version, f_provider_name=provider_name) - for component_name, component_info in version_register_info.get("components").items(): - component_registry_info = { - "f_version": version, - "f_provider_name": provider_name, - "f_component_name": component_name, - "f_module": component_info.get("module") - } - cls.safe_save(ComponentRegistryInfo, component_registry_info, f_version=version, - f_provider_name=provider_name, f_component_name=component_name) - - for component_name, info in cls.REGISTRY["components"].items(): - component_info = { - "f_component_name": component_name, - "f_default_provider": info.get("default_provider"), - "f_support_provider": info.get("support_provider"), - "f_component_alias": info.get("alias"), - } - cls.safe_save(ComponentInfo, component_info, f_component_name=component_name) - - @classmethod - def safe_save(cls, model, defaults, **kwargs): - entity_model, status = model.get_or_create( - **kwargs, - defaults=defaults) - if status is False: - for key in defaults: - setattr(entity_model, key, defaults[key]) - entity_model.save(force_insert=False) - - @classmethod - @DB.connection_context() - def get_from_db(cls, component_registry): - # get component registry info - component_list = ComponentInfo.select() - for component in component_list: - component_registry["components"][component.f_component_name] = { - "default_provider": component.f_default_provider, - "support_provider": component.f_support_provider, - "alias": component.f_component_alias - } - for component_alias in component.f_component_alias: - component_registry["components"][component_alias] = component_registry["components"][component.f_component_name] - - provider_list = ComponentProviderInfo.select() - - # get key names from `fateflow/conf/component_registry.json` - default_version_keys = { - provider_name: default_settings["default_version_key"] - for provider_name, default_settings in component_registry["default_settings"].items() - if "default_version_key" in default_settings - } - - for provider_info in provider_list: - if provider_info.f_provider_name not in component_registry["providers"]: - component_registry["providers"][provider_info.f_provider_name] = { - "default": { - "version": get_versions()[default_version_keys[provider_info.f_provider_name]] - if provider_info.f_provider_name in default_version_keys else provider_info.f_version, - } - } - - component_registry["providers"][provider_info.f_provider_name][provider_info.f_version] = { - "path": provider_info.f_path, - "python": provider_info.f_python, - "class_path": provider_info.f_class_path - } - modules_list = ComponentRegistryInfo.select().where( - ComponentRegistryInfo.f_provider_name == provider_info.f_provider_name, - ComponentRegistryInfo.f_version == provider_info.f_version - ) - modules = {} - for module in modules_list: - modules[module.f_component_name] = {"module": module.f_module} - for component_alias in component_registry["components"][module.f_component_name]["alias"]: - modules[component_alias] = modules[module.f_component_name] - component_registry["providers"][provider_info.f_provider_name][provider_info.f_version]["components"] = modules - return component_registry - - @classmethod - def get_providers(cls): - return cls.REGISTRY.get("providers", {}) - - @classmethod - def get_components(cls): - return cls.REGISTRY.get("components", {}) - - @classmethod - def get_provider_components(cls, provider_name, provider_version): - return cls.get_providers()[provider_name][provider_version]["components"] - - @classmethod - def get_default_class_path(cls): - return ComponentRegistry.REGISTRY["default_settings"]["class_path"] - diff --git a/python/fate_flow/db/db_models.py b/python/fate_flow/db/db_models.py index d2ccbb4bb..5fe351ae9 100644 --- a/python/fate_flow/db/db_models.py +++ b/python/fate_flow/db/db_models.py @@ -14,188 +14,37 @@ # limitations under the License. # import datetime -import inspect -import os -import sys -from functools import wraps - -from peewee import ( - BigAutoField, BigIntegerField, BooleanField, CharField, - CompositeKey, Insert, IntegerField, TextField, -) -from playhouse.hybrid import hybrid_property -from playhouse.pool import PooledMySQLDatabase - -from fate_arch.common import file_utils -from fate_arch.metastore.base_model import ( - BaseModel, DateTimeField, JSONField, ListField, - LongTextField, SerializedField, SerializedType, -) -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.settings import DATABASE, IS_STANDALONE, stat_logger -from fate_flow.utils.log_utils import getLogger -from fate_flow.utils.object_utils import from_dict_hook - - -LOGGER = getLogger() - - -class JsonSerializedField(SerializedField): - def __init__(self, object_hook=from_dict_hook, object_pairs_hook=None, **kwargs): - super(JsonSerializedField, self).__init__(serialized_type=SerializedType.JSON, object_hook=object_hook, - object_pairs_hook=object_pairs_hook, **kwargs) - - -def singleton(cls, *args, **kw): - instances = {} - - def _singleton(): - key = str(cls) + str(os.getpid()) - if key not in instances: - instances[key] = cls(*args, **kw) - return instances[key] - - return _singleton - - -@singleton -class BaseDataBase: - def __init__(self): - database_config = DATABASE.copy() - db_name = database_config.pop("name") - if IS_STANDALONE and not bool(int(os.environ.get("FORCE_USE_MYSQL", 0))): - # sqlite does not support other options - Insert.on_conflict = lambda self, *args, **kwargs: self.on_conflict_replace() - - from playhouse.apsw_ext import APSWDatabase - self.database_connection = APSWDatabase(file_utils.get_project_base_directory("fate_sqlite.db")) - RuntimeConfig.init_config(USE_LOCAL_DATABASE=True) - stat_logger.info('init sqlite database on standalone mode successfully') - else: - self.database_connection = PooledMySQLDatabase(db_name, **database_config) - stat_logger.info('init mysql database on cluster mode successfully') - - -class DatabaseLock: - def __init__(self, lock_name, timeout=10, db=None): - self.lock_name = lock_name - self.timeout = int(timeout) - self.db = db if db else DB - - def lock(self): - # SQL parameters only support %s format placeholders - cursor = self.db.execute_sql("SELECT GET_LOCK(%s, %s)", (self.lock_name, self.timeout)) - ret = cursor.fetchone() - if ret[0] == 0: - raise Exception(f'acquire mysql lock {self.lock_name} timeout') - elif ret[0] == 1: - return True - else: - raise Exception(f'failed to acquire lock {self.lock_name}') - - def unlock(self): - cursor = self.db.execute_sql("SELECT RELEASE_LOCK(%s)", (self.lock_name, )) - ret = cursor.fetchone() - if ret[0] == 0: - raise Exception(f'mysql lock {self.lock_name} was not established by this thread') - elif ret[0] == 1: - return True - else: - raise Exception(f'mysql lock {self.lock_name} does not exist') - - def __enter__(self): - if isinstance(self.db, PooledMySQLDatabase): - self.lock() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if isinstance(self.db, PooledMySQLDatabase): - self.unlock() - - def __call__(self, func): - @wraps(func) - def magic(*args, **kwargs): - with self: - return func(*args, **kwargs) - return magic - - -DB = BaseDataBase().database_connection -DB.lock = DatabaseLock - - -def close_connection(): - try: - if DB: - DB.close() - except Exception as e: - LOGGER.exception(e) - - -class DataBaseModel(BaseModel): - class Meta: - database = DB - - -@DB.connection_context() -def init_database_tables(): - members = inspect.getmembers(sys.modules[__name__], inspect.isclass) - table_objs = [] - create_failed_list = [] - for name, obj in members: - if obj != DataBaseModel and issubclass(obj, DataBaseModel): - table_objs.append(obj) - LOGGER.info(f"start create table {obj.__name__}") - try: - obj.create_table() - LOGGER.info(f"create table success: {obj.__name__}") - except Exception as e: - LOGGER.exception(e) - create_failed_list.append(obj.__name__) - if create_failed_list: - LOGGER.info(f"create tables failed: {create_failed_list}") - raise Exception(f"create tables failed: {create_failed_list}") - - -def fill_db_model_object(model_object, human_model_dict): - for k, v in human_model_dict.items(): - attr_name = 'f_%s' % k - if hasattr(model_object.__class__, attr_name): - setattr(model_object, attr_name, v) - return model_object + +from peewee import CharField, TextField, BigIntegerField, IntegerField, BooleanField, CompositeKey, BigAutoField +from fate_flow.db.base_models import DataBaseModel, JSONField +from fate_flow.entity.types import PROTOCOL class Job(DataBaseModel): - # multi-party common configuration - f_user_id = CharField(max_length=25, null=True) + f_protocol = CharField(max_length=50, default=PROTOCOL.FATE_FLOW) + f_flow_id = CharField(max_length=25, default='') f_job_id = CharField(max_length=25, index=True) - f_name = CharField(max_length=500, null=True, default='') + f_user_name = CharField(max_length=500, null=True, default='') f_description = TextField(null=True, default='') f_tag = CharField(max_length=50, null=True, default='') - f_dsl = JSONField() - f_runtime_conf = JSONField() - f_runtime_conf_on_party = JSONField() - f_train_runtime_conf = JSONField(null=True) - f_roles = JSONField() - f_initiator_role = CharField(max_length=50) + f_dag = JSONField() + f_parties = JSONField() + f_initiator_party_id = CharField(max_length=50) + f_scheduler_party_id = CharField(max_length=50) f_status = CharField(max_length=50) f_status_code = IntegerField(null=True) - f_user = JSONField() + + f_inheritance = JSONField(null=True) + # this party configuration f_role = CharField(max_length=50, index=True) - f_party_id = CharField(max_length=10, index=True) - f_is_initiator = BooleanField(null=True, default=False) + f_party_id = CharField(max_length=50, index=True) f_progress = IntegerField(null=True, default=0) - f_ready_signal = BooleanField(default=False) - f_ready_time = BigIntegerField(null=True) - f_cancel_signal = BooleanField(default=False) - f_cancel_time = BigIntegerField(null=True) - f_rerun_signal = BooleanField(default=False) - f_end_scheduling_updates = IntegerField(null=True, default=0) + f_model_id = CharField(max_length=100, null=True) + f_model_version = CharField(max_length=10) f_engine_name = CharField(max_length=50, null=True) - f_engine_type = CharField(max_length=10, null=True) f_cores = IntegerField(default=0) f_memory = IntegerField(default=0) # MB f_remaining_cores = IntegerField(default=0) @@ -204,13 +53,8 @@ class Job(DataBaseModel): f_apply_resource_time = BigIntegerField(null=True) f_return_resource_time = BigIntegerField(null=True) - f_inheritance_info = JSONField(null=True) - f_inheritance_status = CharField(max_length=50, null=True) - f_start_time = BigIntegerField(null=True) - f_start_date = DateTimeField(null=True) f_end_time = BigIntegerField(null=True) - f_end_date = DateTimeField(null=True) f_elapsed = BigIntegerField(null=True) class Meta: @@ -219,43 +63,42 @@ class Meta: class Task(DataBaseModel): - # multi-party common configuration + f_protocol = CharField(max_length=50, default=PROTOCOL.FATE_FLOW) f_job_id = CharField(max_length=25, index=True) - f_component_name = TextField() - f_component_module = CharField(max_length=200) + f_role = CharField(max_length=50, index=True) + f_party_id = CharField(max_length=50, index=True) + f_task_name = CharField(max_length=50) + f_component = CharField(max_length=50) f_task_id = CharField(max_length=100) f_task_version = BigIntegerField() - f_initiator_role = CharField(max_length=50) - f_initiator_party_id = CharField(max_length=50, default=-1) - f_federated_mode = CharField(max_length=10) - f_federated_status_collect_type = CharField(max_length=10) + f_execution_id = CharField(max_length=100) + f_scheduler_party_id = CharField(max_length=50) f_status = CharField(max_length=50, index=True) f_status_code = IntegerField(null=True) - f_auto_retries = IntegerField(default=0) - f_auto_retry_delay = IntegerField(default=0) - # this party configuration - f_role = CharField(max_length=50, index=True) - f_party_id = CharField(max_length=10, index=True) - f_run_on_this_party = BooleanField(null=True, index=True, default=False) + f_component_parameters = JSONField(null=True) + f_task_run = JSONField(null=True) + f_memory = IntegerField(default=0) + f_task_cores = IntegerField(default=0) + f_resource_in_use = BooleanField(default=False) + f_worker_id = CharField(null=True, max_length=100) f_cmd = JSONField(null=True) f_run_ip = CharField(max_length=100, null=True) f_run_port = IntegerField(null=True) f_run_pid = IntegerField(null=True) f_party_status = CharField(max_length=50) - f_provider_info = JSONField() - f_component_parameters = JSONField() + f_provider_name = CharField(max_length=50) + f_task_parameters = JSONField(null=True) f_engine_conf = JSONField(null=True) f_kill_status = BooleanField(default=False) f_error_report = TextField(default="") + f_sync_type = CharField(max_length=20) + f_timeout = IntegerField(null=True) - f_is_deepspeed = BooleanField(default=False) - f_deepspeed_id = CharField(max_length=200, null=True) + f_launcher_name = CharField(max_length=20, null=True) f_start_time = BigIntegerField(null=True) - f_start_date = DateTimeField(null=True) f_end_time = BigIntegerField(null=True) - f_end_date = DateTimeField(null=True) f_elapsed = BigIntegerField(null=True) class Meta: @@ -263,169 +106,60 @@ class Meta: primary_key = CompositeKey('f_job_id', 'f_task_id', 'f_task_version', 'f_role', 'f_party_id') -class TrackingMetric(DataBaseModel): - _mapper = {} - - @classmethod - def model(cls, table_index=None, date=None): - if not table_index: - table_index = date.strftime( - '%Y%m%d') if date else datetime.datetime.now().strftime( - '%Y%m%d') - class_name = 'TrackingMetric_%s' % table_index - - ModelClass = TrackingMetric._mapper.get(class_name, None) - if ModelClass is None: - class Meta: - db_table = '%s_%s' % ('t_tracking_metric', table_index) - - attrs = {'__module__': cls.__module__, 'Meta': Meta} - ModelClass = type("%s_%s" % (cls.__name__, table_index), (cls,), - attrs) - TrackingMetric._mapper[class_name] = ModelClass - return ModelClass() - - f_id = BigAutoField(primary_key=True) +class TrackingOutputInfo(DataBaseModel): f_job_id = CharField(max_length=25, index=True) - f_component_name = CharField(max_length=30, index=True) - f_task_id = CharField(max_length=100, null=True) - f_task_version = BigIntegerField(null=True) - f_role = CharField(max_length=10, index=True) - f_party_id = CharField(max_length=10) - f_metric_namespace = CharField(max_length=80, index=True) - f_metric_name = CharField(max_length=80, index=True) - f_key = CharField(max_length=200) - f_value = LongTextField() - f_type = IntegerField() # 0 is data, 1 is meta - - -class TrackingOutputDataInfo(DataBaseModel): - _mapper = {} - - @classmethod - def model(cls, table_index=None, date=None): - if not table_index: - table_index = date.strftime( - '%Y%m%d') if date else datetime.datetime.now().strftime( - '%Y%m%d') - class_name = 'TrackingOutputDataInfo_%s' % table_index - - ModelClass = TrackingOutputDataInfo._mapper.get(class_name, None) - if ModelClass is None: - class Meta: - db_table = '%s_%s' % ('t_tracking_output_data_info', table_index) - primary_key = CompositeKey( - 'f_job_id', 'f_task_id', 'f_task_version', - 'f_data_name', 'f_role', 'f_party_id', - ) - - attrs = {'__module__': cls.__module__, 'Meta': Meta} - ModelClass = type("%s_%s" % (cls.__name__, table_index), (cls,), - attrs) - TrackingOutputDataInfo._mapper[class_name] = ModelClass - return ModelClass() - - # multi-party common configuration - f_job_id = CharField(max_length=25, index=True) - f_component_name = TextField() f_task_id = CharField(max_length=100, null=True, index=True) f_task_version = BigIntegerField(null=True) - f_data_name = CharField(max_length=30) - # this party configuration + f_task_name = CharField(max_length=50, index=True) f_role = CharField(max_length=50, index=True) - f_party_id = CharField(max_length=10, index=True) - f_table_name = CharField(max_length=500, null=True) - f_table_namespace = CharField(max_length=500, null=True) - f_description = TextField(null=True, default='') - - -class MachineLearningModelInfo(DataBaseModel): - f_role = CharField(max_length=50) - f_party_id = CharField(max_length=10) - f_roles = JSONField(default={}) - f_job_id = CharField(max_length=25, index=True) - f_model_id = CharField(max_length=100, index=True) - f_model_version = CharField(max_length=100, index=True) - f_size = BigIntegerField(default=0) - f_initiator_role = CharField(max_length=50) - f_initiator_party_id = CharField(max_length=50, default=-1) - # TODO: deprecated. use f_train_runtime_conf instead - f_runtime_conf = JSONField(default={}) - f_train_dsl = JSONField(default={}) - f_train_runtime_conf = JSONField(default={}) - f_runtime_conf_on_party = JSONField(default={}) - f_inference_dsl = JSONField(default={}) - f_fate_version = CharField(max_length=10, null=True, default='') - f_parent = BooleanField(null=True, default=None) - f_parent_info = JSONField(default={}) - # loaded times in api /model/load/do - f_loaded_times = IntegerField(default=0) - # imported from api /model/import - f_imported = IntegerField(default=0) - f_archive_sha256 = CharField(max_length=100, null=True) - f_archive_from_ip = CharField(max_length=100, null=True) - - @hybrid_property - def f_party_model_id(self): - return '#'.join([self.f_role, self.f_party_id, self.f_model_id]) - - class Meta: - db_table = "t_machine_learning_model_info" - primary_key = CompositeKey('f_role', 'f_party_id', 'f_model_id', 'f_model_version') - - -class DataTableTracking(DataBaseModel): - f_table_id = BigAutoField(primary_key=True) - f_table_name = CharField(max_length=300, null=True) - f_table_namespace = CharField(max_length=300, null=True) - f_job_id = CharField(max_length=25, index=True, null=True) - f_have_parent = BooleanField(default=False) - f_parent_number = IntegerField(default=0) - - f_parent_table_name = CharField(max_length=500, null=True) - f_parent_table_namespace = CharField(max_length=500, null=True) - f_source_table_name = CharField(max_length=500, null=True) - f_source_table_namespace = CharField(max_length=500, null=True) - - class Meta: - db_table = "t_data_table_tracking" - - -class CacheRecord(DataBaseModel): - f_cache_key = CharField(max_length=500) - f_cache = JsonSerializedField() - f_job_id = CharField(max_length=25, index=True, null=True) - f_role = CharField(max_length=50, index=True, null=True) - f_party_id = CharField(max_length=10, index=True, null=True) - f_component_name = TextField(null=True) - f_task_id = CharField(max_length=100, null=True) - f_task_version = BigIntegerField(null=True, index=True) - f_cache_name = CharField(max_length=50, null=True) - t_ttl = BigIntegerField(default=0) + f_party_id = CharField(max_length=50, index=True) + f_output_key = CharField(max_length=30) + f_index = IntegerField() + f_uri = CharField(max_length=200, null=True) + f_namespace = CharField(max_length=200) + f_name = CharField(max_length=200) class Meta: - db_table = "t_cache_record" + db_table = "t_tracking_data_output" + primary_key = CompositeKey('f_job_id', 'f_task_id', 'f_task_version', 'f_role', 'f_party_id', 'f_output_key', 'f_uri') -class ModelTag(DataBaseModel): - f_id = BigAutoField(primary_key=True) - f_m_id = CharField(max_length=25, null=False) - f_t_id = BigIntegerField(null=False) +class EngineRegistry(DataBaseModel): + f_engine_type = CharField(max_length=10, index=True) + f_engine_name = CharField(max_length=50, index=True) + f_engine_config = JSONField() + f_cores = IntegerField() + f_memory = IntegerField() # MB + f_remaining_cores = IntegerField() + f_remaining_memory = IntegerField() # MB class Meta: - db_table = "t_model_tag" + db_table = "t_engine_registry" + primary_key = CompositeKey('f_engine_name', 'f_engine_type') -class Tag(DataBaseModel): - f_id = BigAutoField(primary_key=True) - f_name = CharField(max_length=100, unique=True) - f_desc = TextField(null=True) +class WorkerInfo(DataBaseModel): + f_worker_id = CharField(max_length=100, primary_key=True) + f_worker_name = CharField(max_length=50, index=True) + f_job_id = CharField(max_length=25, index=True) + f_task_id = CharField(max_length=100) + f_task_version = BigIntegerField(index=True) + f_role = CharField(max_length=50) + f_party_id = CharField(max_length=50, index=True) + f_run_ip = CharField(max_length=100, null=True) + f_run_pid = IntegerField(null=True) + f_http_port = IntegerField(null=True) + f_grpc_port = IntegerField(null=True) + f_config = JSONField(null=True) + f_cmd = JSONField(null=True) + f_start_time = BigIntegerField(null=True) + f_end_time = BigIntegerField(null=True) class Meta: - db_table = "t_tags" + db_table = "t_worker" -class ComponentSummary(DataBaseModel): +class Metric(DataBaseModel): _mapper = {} @classmethod @@ -434,115 +168,75 @@ def model(cls, table_index=None, date=None): table_index = date.strftime( '%Y%m%d') if date else datetime.datetime.now().strftime( '%Y%m%d') - class_name = 'ComponentSummary_%s' % table_index + class_name = 'Metric_%s' % table_index - ModelClass = TrackingMetric._mapper.get(class_name, None) + ModelClass = Metric._mapper.get(class_name, None) if ModelClass is None: class Meta: - db_table = '%s_%s' % ('t_component_summary', table_index) + db_table = '%s_%s' % ('t_tracking_metric', table_index) attrs = {'__module__': cls.__module__, 'Meta': Meta} - ModelClass = type("%s_%s" % (cls.__name__, table_index), (cls,), attrs) - ComponentSummary._mapper[class_name] = ModelClass + ModelClass = type("%s_%s" % (cls.__name__, table_index), (cls,), + attrs) + Metric._mapper[class_name] = ModelClass return ModelClass() f_id = BigAutoField(primary_key=True) f_job_id = CharField(max_length=25, index=True) - f_role = CharField(max_length=25, index=True) - f_party_id = CharField(max_length=10, index=True) - f_component_name = CharField(max_length=50) - f_task_id = CharField(max_length=50, null=True, index=True) - f_task_version = CharField(max_length=50, null=True) - f_summary = LongTextField() - - -class EngineRegistry(DataBaseModel): - f_engine_type = CharField(max_length=10, index=True) - f_engine_name = CharField(max_length=50, index=True) - f_engine_entrance = CharField(max_length=50, index=True) - f_engine_config = JSONField() - f_cores = IntegerField() - f_memory = IntegerField() # MB - f_remaining_cores = IntegerField() - f_remaining_memory = IntegerField() # MB - f_nodes = IntegerField() - - class Meta: - db_table = "t_engine_registry" - primary_key = CompositeKey('f_engine_name', 'f_engine_type') - - -# component registry -class ComponentRegistryInfo(DataBaseModel): - f_provider_name = CharField(max_length=20, index=True) - f_version = CharField(max_length=10, index=True) - f_component_name = CharField(max_length=30, index=True) - f_module = CharField(max_length=128) - - class Meta: - db_table = "t_component_registry" - primary_key = CompositeKey('f_provider_name', 'f_version', 'f_component_name') + f_role = CharField(max_length=10, index=True) + f_party_id = CharField(max_length=50) + f_task_name = CharField(max_length=50, index=True) + f_task_id = CharField(max_length=100) + f_task_version = BigIntegerField(null=True) + f_name = CharField(max_length=30, index=True) + f_type = CharField(max_length=30, index=True, null=True) + f_groups = JSONField() + f_step_axis = CharField(max_length=30, index=True, null=True) + f_data = JSONField() -class ComponentProviderInfo(DataBaseModel): - f_provider_name = CharField(max_length=20, index=True) - f_version = CharField(max_length=10, index=True) - f_class_path = JSONField() - f_path = CharField(max_length=128, null=False) - f_python = CharField(max_length=128, null=False) +class ProviderInfo(DataBaseModel): + f_provider_name = CharField(max_length=100, primary_key=True) + f_name = CharField(max_length=20, index=True) + f_version = CharField(max_length=20) + f_device = CharField(max_length=20) + f_metadata = JSONField() class Meta: - db_table = "t_component_provider_info" - primary_key = CompositeKey('f_provider_name', 'f_version') + db_table = "t_provider_info" class ComponentInfo(DataBaseModel): - f_component_name = CharField(max_length=30, primary_key=True) - f_component_alias = JSONField() - f_default_provider = CharField(max_length=20) - f_support_provider = ListField(null=True) + f_provider_name = CharField(max_length=100) + f_protocol = CharField(max_length=20, default=PROTOCOL.FATE_FLOW) + f_name = CharField(max_length=20, index=True) + f_version = CharField(max_length=20) + f_device = CharField(max_length=20) + f_component_name = CharField(max_length=50) + f_component_entrypoint = JSONField(null=True) + f_component_description = JSONField(null=True) class Meta: db_table = "t_component_info" + primary_key = CompositeKey("f_provider_name", "f_component_name", "f_protocol") -class WorkerInfo(DataBaseModel): - f_worker_id = CharField(max_length=100, primary_key=True) - f_worker_name = CharField(max_length=50, index=True) +class PipelineModelMeta(DataBaseModel): + f_model_id = CharField(max_length=100) + f_model_version = CharField(max_length=10) f_job_id = CharField(max_length=25, index=True) - f_task_id = CharField(max_length=100) - f_task_version = BigIntegerField(index=True) - f_role = CharField(max_length=50) - f_party_id = CharField(max_length=10, index=True) - f_run_ip = CharField(max_length=100, null=True) - f_run_pid = IntegerField(null=True) - f_http_port = IntegerField(null=True) - f_grpc_port = IntegerField(null=True) - f_config = JSONField(null=True) - f_cmd = JSONField(null=True) - f_start_time = BigIntegerField(null=True) - f_start_date = DateTimeField(null=True) - f_end_time = BigIntegerField(null=True) - f_end_date = DateTimeField(null=True) - - class Meta: - db_table = "t_worker" - - -class DependenciesStorageMeta(DataBaseModel): - f_storage_engine = CharField(max_length=30) - f_type = CharField(max_length=20) - f_version = CharField(max_length=10, index=True) - f_storage_path = CharField(max_length=256, null=True) - f_snapshot_time = BigIntegerField(null=True) - f_fate_flow_snapshot_time = BigIntegerField(null=True) - f_dependencies_conf = JSONField(null=True) - f_upload_status = BooleanField(default=False) - f_pid = IntegerField(null=True) + f_role = CharField(max_length=50, index=True) + f_party_id = CharField(max_length=50, index=True) + f_task_name = CharField(max_length=50, index=True) + f_storage_key = CharField(max_length=100) + f_output_key = CharField(max_length=20) + f_type_name = CharField(max_length=20) + f_meta_data = JSONField(null=True) + f_storage_engine = CharField(max_length=30, null=True, index=True) class Meta: - db_table = "t_dependencies_storage_meta" - primary_key = CompositeKey('f_storage_engine', 'f_type', 'f_version') + db_table = 't_model_meta' + primary_key = CompositeKey('f_job_id', 'f_storage_key', "f_storage_engine") class ServerRegistryInfo(DataBaseModel): @@ -552,7 +246,7 @@ class ServerRegistryInfo(DataBaseModel): f_protocol = CharField(max_length=10) class Meta: - db_table = "t_server_registry_info" + db_table = "t_server" class ServiceRegistryInfo(DataBaseModel): @@ -565,35 +259,5 @@ class ServiceRegistryInfo(DataBaseModel): f_headers = JSONField(null=True) class Meta: - db_table = "t_service_registry_info" + db_table = "t_service" primary_key = CompositeKey('f_server_name', 'f_service_name') - - -class SiteKeyInfo(DataBaseModel): - f_party_id = CharField(max_length=10, index=True) - f_key_name = CharField(max_length=10, index=True) - f_key = LongTextField() - - class Meta: - db_table = "t_site_key_info" - primary_key = CompositeKey('f_party_id', 'f_key_name') - - -class PipelineComponentMeta(DataBaseModel): - f_model_id = CharField(max_length=100, index=True) - f_model_version = CharField(max_length=100, index=True) - f_role = CharField(max_length=50, index=True) - f_party_id = CharField(max_length=10, index=True) - f_component_name = CharField(max_length=100, index=True) - f_component_module_name = CharField(max_length=100) - f_model_alias = CharField(max_length=100, index=True) - f_model_proto_index = JSONField(null=True) - f_run_parameters = JSONField(null=True) - f_archive_sha256 = CharField(max_length=100, null=True) - f_archive_from_ip = CharField(max_length=100, null=True) - - class Meta: - db_table = 't_pipeline_component_meta' - indexes = ( - (('f_model_id', 'f_model_version', 'f_role', 'f_party_id', 'f_component_name'), True), - ) diff --git a/python/fate_flow/db/db_utils.py b/python/fate_flow/db/db_utils.py deleted file mode 100644 index 837d7c04e..000000000 --- a/python/fate_flow/db/db_utils.py +++ /dev/null @@ -1,139 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import operator -from functools import reduce -from typing import Dict, Type, Union - -from fate_arch.common.base_utils import current_timestamp, timestamp_to_date - -from fate_flow.db.db_models import DB, DataBaseModel -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.utils.log_utils import getLogger - - -LOGGER = getLogger() - - -@DB.connection_context() -def bulk_insert_into_db(model, data_source, replace_on_conflict=False): - DB.create_tables([model]) - - current_time = current_timestamp() - current_date = timestamp_to_date(current_time) - - for data in data_source: - if 'f_create_time' not in data: - data['f_create_time'] = current_time - data['f_create_date'] = timestamp_to_date(data['f_create_time']) - data['f_update_time'] = current_time - data['f_update_date'] = current_date - - preserve = tuple(data_source[0].keys() - {'f_create_time', 'f_create_date'}) - - batch_size = 50 if RuntimeConfig.USE_LOCAL_DATABASE else 1000 - - for i in range(0, len(data_source), batch_size): - with DB.atomic(): - query = model.insert_many(data_source[i:i + batch_size]) - if replace_on_conflict: - query = query.on_conflict(preserve=preserve) - query.execute() - - -def get_dynamic_db_model(base, job_id): - return type(base.model(table_index=get_dynamic_tracking_table_index(job_id=job_id))) - - -def get_dynamic_tracking_table_index(job_id): - return job_id[:8] - - -def fill_db_model_object(model_object, human_model_dict): - for k, v in human_model_dict.items(): - attr_name = 'f_%s' % k - if hasattr(model_object.__class__, attr_name): - setattr(model_object, attr_name, v) - return model_object - - -# https://docs.peewee-orm.com/en/latest/peewee/query_operators.html -supported_operators = { - '==': operator.eq, - '<': operator.lt, - '<=': operator.le, - '>': operator.gt, - '>=': operator.ge, - '!=': operator.ne, - '<<': operator.lshift, - '>>': operator.rshift, - '%': operator.mod, - '**': operator.pow, - '^': operator.xor, - '~': operator.inv, -} -''' -query = { - # Job.f_job_id == '1234567890' - 'job_id': '1234567890', - # Job.f_party_id == 999 - 'party_id': 999, - # Job.f_tag != 'submit_failed' - 'tag': ('!=', 'submit_failed'), - # Job.f_status.in_(['success', 'running', 'waiting']) - 'status': ('in_', ['success', 'running', 'waiting']), - # Job.f_create_time.between(10000, 99999) - 'create_time': ('between', 10000, 99999), - # Job.f_description.distinct() - 'description': ('distinct', ), -} -''' -def query_dict2expression(model: Type[DataBaseModel], query: Dict[str, Union[bool, int, str, list, tuple]]): - expression = [] - - for field, value in query.items(): - if not isinstance(value, (list, tuple)): - value = ('==', value) - op, *val = value - - field = getattr(model, f'f_{field}') - value = supported_operators[op](field, val[0]) if op in supported_operators else getattr(field, op)(*val) - expression.append(value) - - return reduce(operator.iand, expression) - - -def query_db(model: Type[DataBaseModel], limit: int = 0, offset: int = 0, - query: dict = None, order_by: Union[str, list, tuple] = None): - data = model.select() - if query: - data = data.where(query_dict2expression(model, query)) - count = data.count() - - if not order_by: - order_by = 'create_time' - if not isinstance(order_by, (list, tuple)): - order_by = (order_by, 'asc') - order_by, order = order_by - order_by = getattr(model, f'f_{order_by}') - order_by = getattr(order_by, order)() - data = data.order_by(order_by) - - if limit > 0: - data = data.limit(limit) - if offset > 0: - data = data.offset(offset) - - return list(data), count diff --git a/python/fate_flow/db/dependence_registry.py b/python/fate_flow/db/dependence_registry.py deleted file mode 100644 index c1f971600..000000000 --- a/python/fate_flow/db/dependence_registry.py +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os - -from fate_flow.db.db_models import DependenciesStorageMeta, DB -from fate_flow.entity.types import FateDependenceStorageEngine - - -class DependenceRegistry: - @classmethod - @DB.connection_context() - def get_dependencies_storage_meta(cls, get_or_one=False, **kwargs): - kwargs["storage_engine"] = FateDependenceStorageEngine.HDFS.value - dependencies_storage_info = DependenciesStorageMeta.query(**kwargs) - if get_or_one: - return dependencies_storage_info[0] if dependencies_storage_info else None - return dependencies_storage_info - - @classmethod - @DB.connection_context() - def save_dependencies_storage_meta(cls, storage_meta, status_check=False): - entity_model, status = DependenciesStorageMeta.get_or_create( - f_storage_engine=storage_meta.get("f_storage_engine"), - f_type=storage_meta.get("f_type"), - f_version=storage_meta.get("f_version"), - defaults=storage_meta) - if status is False: - if status_check: - if "f_upload_status" in storage_meta.keys() and storage_meta["f_upload_status"] \ - != entity_model.f_upload_status: - return - for key in storage_meta: - setattr(entity_model, key, storage_meta[key]) - entity_model.save(force_insert=False) - - @classmethod - def get_modify_time(cls, path): - return int(os.path.getmtime(path)*1000) - - diff --git a/python/fate_flow/db/fate_casbin.py b/python/fate_flow/db/fate_casbin.py deleted file mode 100644 index b09f01427..000000000 --- a/python/fate_flow/db/fate_casbin.py +++ /dev/null @@ -1,67 +0,0 @@ -import casbin -import casbin_sqlalchemy_adapter -import pymysql - -from fate_flow.db.db_models import singleton -from fate_flow.settings import CASBIN_MODEL_CONF, DATABASE, CASBIN_TABLE_NAME, PERMISSION_SWITCH -from sqlalchemy import Column, Integer, String, TEXT, create_engine - -pymysql.install_as_MySQLdb() - -class FateCasbinRule(casbin_sqlalchemy_adapter.Base): - __tablename__ = CASBIN_TABLE_NAME - - id = Column(Integer, primary_key=True) - ptype = Column(String(255)) - v0 = Column(String(255)) - v1 = Column(String(255)) - v2 = Column(TEXT()) - v3 = Column(Integer()) - v4 = Column(String(255)) - v5 = Column(String(255)) - - def __str__(self): - arr = [self.ptype] - for v in (self.v0, self.v1, self.v2, self.v3, self.v4, self.v5): - if v is None: - break - arr.append(v) - return ", ".join(arr) - - def __repr__(self): - return ''.format(self.id, str(self)) - - -@singleton -class FateCasbin(): - def __init__(self): - self.engine = create_engine( - f"mysql://{DATABASE.get('user')}:{DATABASE.get('passwd')}@{DATABASE.get('host')}:{DATABASE.get('port')}/{DATABASE.get('name')}") - self.adapter = casbin_sqlalchemy_adapter.Adapter(self.engine, FateCasbinRule) - self.e = casbin.Enforcer(CASBIN_MODEL_CONF, self.adapter) - - def query(self, party_id): - self.e.load_policy() - return self.e.get_permissions_for_user(party_id) - - def delete(self, party_id, type, value): - self.e.load_policy() - return self.e.delete_permission_for_user(party_id, type, value) - - def delete_all(self, party_id, type): - self.e.load_policy() - return self.e.remove_filtered_policy(0, party_id, type) - - def grant(self, party_id, type, value): - self.e.load_policy() - return self.e.add_permission_for_user(party_id, type, value) - - def enforce(self, party_id, type, value): - self.e.load_policy() - try: - return self.e.enforce(party_id, type, str(value)) - except Exception as e: - raise Exception(f"{party_id}, {type}, {value} {e}") - - -CB = FateCasbin() if PERMISSION_SWITCH else None \ No newline at end of file diff --git a/python/fate_flow/db/key_manager.py b/python/fate_flow/db/key_manager.py deleted file mode 100644 index 8fd8c46b3..000000000 --- a/python/fate_flow/db/key_manager.py +++ /dev/null @@ -1,67 +0,0 @@ -from Crypto.PublicKey import RSA -from Crypto import Random - -from fate_flow.db.db_models import DB, SiteKeyInfo -from fate_flow.entity.types import SiteKeyName -from fate_flow.settings import SITE_AUTHENTICATION, PARTY_ID - - -def rsa_key_generate(): - random_generator = Random.new().read - rsa = RSA.generate(2048, random_generator) - private_pem = rsa.exportKey().decode() - public_pem = rsa.publickey().exportKey().decode() - return private_pem, public_pem - - -class RsaKeyManager: - @classmethod - def init(cls): - if PARTY_ID and SITE_AUTHENTICATION: - if not cls.get_key(PARTY_ID, key_name=SiteKeyName.PRIVATE.value): - cls.generate_key(PARTY_ID) - - @classmethod - @DB.connection_context() - def create_or_update(cls, party_id, key, key_name=SiteKeyName.PUBLIC.value): - defaults = { - "f_party_id": party_id, - "f_key_name": key_name, - "f_key": key - } - entity_model, status = SiteKeyInfo.get_or_create( - f_party_id=party_id, - f_key_name=key_name, - defaults=defaults - ) - if status is False: - for key in defaults: - setattr(entity_model, key, defaults[key]) - entity_model.save(force_insert=False) - return "update success" - else: - return "save success" - - @classmethod - def generate_key(cls, party_id): - private_key, public_key = rsa_key_generate() - cls.create_or_update(party_id, private_key, key_name=SiteKeyName.PRIVATE.value) - cls.create_or_update(party_id, public_key, key_name=SiteKeyName.PUBLIC.value) - - @classmethod - @DB.connection_context() - def get_key(cls, party_id, key_name=SiteKeyName.PUBLIC.value): - site_info = SiteKeyInfo.query(party_id=party_id, key_name=key_name) - if site_info: - return site_info[0].f_key - else: - return None - - @classmethod - @DB.connection_context() - def delete(cls, party_id, key_name=SiteKeyName.PUBLIC.value): - site_info = SiteKeyInfo.query(party_id=party_id, key_name=key_name) - if site_info: - return site_info[0].delete_instance() - else: - return None diff --git a/python/fate_flow/db/permission_models.py b/python/fate_flow/db/permission_models.py new file mode 100644 index 000000000..090c15510 --- /dev/null +++ b/python/fate_flow/db/permission_models.py @@ -0,0 +1,37 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from peewee import CharField, CompositeKey + +from fate_flow.db import DataBaseModel + + +class AppInfo(DataBaseModel): + f_app_name = CharField(max_length=100, index=True) + f_app_id = CharField(max_length=100, primary_key=True) + f_app_token = CharField(max_length=100) + f_app_type = CharField(max_length=20, index=True) + + class Meta: + db_table = "t_app_info" + + +class PartnerAppInfo(DataBaseModel): + f_party_id = CharField(max_length=100, primary_key=True) + f_app_id = CharField(max_length=100) + f_app_token = CharField(max_length=100) + + class Meta: + db_table = "t_partner_app_info" diff --git a/python/fate_flow/db/schedule_models.py b/python/fate_flow/db/schedule_models.py new file mode 100644 index 000000000..d293107d2 --- /dev/null +++ b/python/fate_flow/db/schedule_models.py @@ -0,0 +1,84 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from peewee import CharField, TextField, IntegerField, BooleanField, BigIntegerField, CompositeKey + +from fate_flow.db.base_models import DataBaseModel, JSONField +from fate_flow.entity.types import PROTOCOL + + +class ScheduleJob(DataBaseModel): + f_protocol = CharField(max_length=50, default=PROTOCOL.FATE_FLOW) + f_job_id = CharField(max_length=25, index=True) + f_priority = IntegerField(default=0) + f_tag = CharField(max_length=50, null=True, default='') + f_dag = JSONField(null=True) + f_parties = JSONField() + f_initiator_party_id = CharField(max_length=50) + f_scheduler_party_id = CharField(max_length=50) + f_status = CharField(max_length=50) + f_status_code = IntegerField(null=True) + + f_progress = IntegerField(null=True, default=0) + f_schedule_signal = BooleanField(default=False) + f_schedule_time = BigIntegerField(null=True) + f_cancel_signal = BooleanField(default=False) + f_cancel_time = BigIntegerField(null=True) + f_rerun_signal = BooleanField(default=False) + f_end_scheduling_updates = IntegerField(null=True, default=0) + + f_start_time = BigIntegerField(null=True) + f_end_time = BigIntegerField(null=True) + f_elapsed = BigIntegerField(null=True) + + class Meta: + db_table = "t_schedule_job" + primary_key = CompositeKey('f_job_id') + + +class ScheduleTask(DataBaseModel): + f_job_id = CharField(max_length=25, index=True) + f_role = CharField(max_length=50, index=True) + f_party_id = CharField(max_length=50, index=True) + f_task_name = CharField(max_length=50) + f_component = CharField(max_length=50) + f_task_id = CharField(max_length=100) + f_task_version = BigIntegerField() + f_parties = JSONField() + f_error_report = TextField(default="") + f_status = CharField(max_length=50) + + f_start_time = BigIntegerField(null=True) + f_end_time = BigIntegerField(null=True) + f_elapsed = BigIntegerField(null=True) + + class Meta: + db_table = "t_schedule_task" + primary_key = CompositeKey('f_job_id', 'f_task_id', 'f_task_version', 'f_role', 'f_party_id') + + +class ScheduleTaskStatus(DataBaseModel): + f_job_id = CharField(max_length=25, index=True) + f_task_name = CharField(max_length=50) + f_task_id = CharField(max_length=100) + f_task_version = BigIntegerField() + f_status = CharField(max_length=50) + f_auto_retries = IntegerField(default=0) + f_sync_type = CharField(max_length=10) + + class Meta: + db_table = "t_schedule_task_status" + primary_key = CompositeKey('f_job_id', 'f_task_id', 'f_task_version') + diff --git a/python/fate_flow/db/service_registry.py b/python/fate_flow/db/service_registry.py deleted file mode 100644 index 183b7644c..000000000 --- a/python/fate_flow/db/service_registry.py +++ /dev/null @@ -1,172 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import socket -from pathlib import Path -from fate_arch.common import file_utils, conf_utils -from fate_arch.common.conf_utils import SERVICE_CONF -from .db_models import DB, ServiceRegistryInfo, ServerRegistryInfo -from .reload_config_base import ReloadConfigBase - - -class ServiceRegistry(ReloadConfigBase): - @classmethod - @DB.connection_context() - def load_service(cls, **kwargs) -> [ServiceRegistryInfo]: - service_registry_list = ServiceRegistryInfo.query(**kwargs) - return [service for service in service_registry_list] - - @classmethod - @DB.connection_context() - def save_service_info(cls, server_name, service_name, uri, method="POST", server_info=None, params=None, data=None, headers=None, protocol="http"): - if not server_info: - server_list = ServerRegistry.query_server_info_from_db(server_name=server_name) - if not server_list: - raise Exception(f"no found server {server_name}") - server_info = server_list[0] - url = f"{server_info.f_protocol}://{server_info.f_host}:{server_info.f_port}{uri}" - else: - url = f"{server_info.get('protocol', protocol)}://{server_info.get('host')}:{server_info.get('port')}{uri}" - service_info = { - "f_server_name": server_name, - "f_service_name": service_name, - "f_url": url, - "f_method": method, - "f_params": params if params else {}, - "f_data": data if data else {}, - "f_headers": headers if headers else {} - } - entity_model, status = ServiceRegistryInfo.get_or_create( - f_server_name=server_name, - f_service_name=service_name, - defaults=service_info) - if status is False: - for key in service_info: - setattr(entity_model, key, service_info[key]) - entity_model.save(force_insert=False) - - -class ServerRegistry(ReloadConfigBase): - FATEBOARD = None - FATE_ON_STANDALONE = None - FATE_ON_EGGROLL = None - FATE_ON_SPARK = None - MODEL_STORE_ADDRESS = None - SERVINGS = None - FATEMANAGER = None - STUDIO = None - - @classmethod - def load(cls): - cls.load_server_info_from_conf() - cls.load_server_info_from_db() - - @classmethod - def load_server_info_from_conf(cls): - path = Path(file_utils.get_project_base_directory()) / 'conf' / SERVICE_CONF - conf = file_utils.load_yaml_conf(path) - if not isinstance(conf, dict): - raise ValueError('invalid config file') - - local_path = path.with_name(f'local.{SERVICE_CONF}') - if local_path.exists(): - local_conf = file_utils.load_yaml_conf(local_path) - if not isinstance(local_conf, dict): - raise ValueError('invalid local config file') - conf.update(local_conf) - for k, v in conf.items(): - if isinstance(v, dict): - setattr(cls, k.upper(), v) - - @classmethod - def register(cls, server_name, server_info): - cls.save_server_info_to_db(server_name, server_info.get("host"), server_info.get("port"), protocol=server_info.get("protocol", "http")) - setattr(cls, server_name, server_info) - - @classmethod - def save(cls, service_config): - update_server = {} - for server_name, server_info in service_config.items(): - cls.parameter_check(server_info) - api_info = server_info.pop("api", {}) - for service_name, info in api_info.items(): - ServiceRegistry.save_service_info( - server_name, service_name, uri=info.get('uri'), - method=info.get('method', 'POST'), - server_info=server_info, - data=info.get("data", {}), - headers=info.get("headers", {}), - params=info.get("params", {}) - ) - cls.save_server_info_to_db(server_name, server_info.get("host"), server_info.get("port"), protocol="http") - setattr(cls, server_name.upper(), server_info) - return update_server - - @classmethod - def parameter_check(cls, service_info): - if "host" in service_info and "port" in service_info: - cls.connection_test(service_info.get("host"), service_info.get("port")) - - @classmethod - def connection_test(cls, ip, port): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = s.connect_ex((ip, port)) - if result != 0: - raise ConnectionRefusedError(f"connection refused: host {ip}, port {port}") - - @classmethod - def query(cls, service_name, default=None): - service_info = getattr(cls, service_name, default) - if not service_info: - service_info = conf_utils.get_base_config(service_name, default) - return service_info - - @classmethod - @DB.connection_context() - def query_server_info_from_db(cls, server_name=None) -> [ServerRegistryInfo]: - if server_name: - server_list = ServerRegistryInfo.select().where(ServerRegistryInfo.f_server_name==server_name.upper()) - else: - server_list = ServerRegistryInfo.select() - return [server for server in server_list] - - @classmethod - @DB.connection_context() - def load_server_info_from_db(cls): - for server in cls.query_server_info_from_db(): - server_info = { - "host": server.f_host, - "port": server.f_port, - "protocol": server.f_protocol - } - setattr(cls, server.f_server_name.upper(), server_info) - - - @classmethod - @DB.connection_context() - def save_server_info_to_db(cls, server_name, host, port, protocol="http"): - server_info = { - "f_server_name": server_name, - "f_host": host, - "f_port": port, - "f_protocol": protocol - } - entity_model, status = ServerRegistryInfo.get_or_create( - f_server_name=server_name, - defaults=server_info) - if status is False: - for key in server_info: - setattr(entity_model, key, server_info[key]) - entity_model.save(force_insert=False) diff --git a/python/fate_flow/db/storage_models.py b/python/fate_flow/db/storage_models.py new file mode 100644 index 000000000..f291ad686 --- /dev/null +++ b/python/fate_flow/db/storage_models.py @@ -0,0 +1,65 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from peewee import CharField, IntegerField, BooleanField, BigIntegerField, TextField, DateTimeField, CompositeKey + +from fate_flow.db.base_models import DataBaseModel, JSONField + + +class StorageConnectorModel(DataBaseModel): + f_name = CharField(max_length=100, primary_key=True) + f_engine = CharField(max_length=100, index=True) # 'MYSQL' + f_connector_info = JSONField() + + class Meta: + db_table = "t_storage_connector" + + +class StorageTableMetaModel(DataBaseModel): + f_name = CharField(max_length=100, index=True) + f_namespace = CharField(max_length=100, index=True) + f_address = JSONField() + f_engine = CharField(max_length=100) # 'EGGROLL', 'MYSQL' + f_options = JSONField() + f_partitions = IntegerField(null=True) + + f_delimiter = CharField(null=True) + f_have_head = BooleanField(default=True) + f_extend_sid = BooleanField(default=False) + f_data_meta = JSONField() + f_count = BigIntegerField(null=True) + f_part_of_data = JSONField() + f_source = JSONField() + f_data_type = CharField(max_length=20, null=True) + f_disable = BooleanField(default=False) + f_description = TextField(default='') + + f_read_access_time = BigIntegerField(null=True) + f_write_access_time = BigIntegerField(null=True) + + class Meta: + db_table = "t_storage_table_meta" + primary_key = CompositeKey('f_name', 'f_namespace') + + +class SessionRecord(DataBaseModel): + f_engine_session_id = CharField(max_length=150, null=False) + f_manager_session_id = CharField(max_length=150, null=False) + f_engine_type = CharField(max_length=10, index=True) + f_engine_name = CharField(max_length=50, index=True) + f_engine_address = JSONField() + + class Meta: + db_table = "t_session_record" + primary_key = CompositeKey("f_engine_type", "f_engine_name", "f_engine_session_id") diff --git a/python/fate_flow/deepspeed_client.py b/python/fate_flow/deepspeed_client.py deleted file mode 100644 index 9c7f7daa0..000000000 --- a/python/fate_flow/deepspeed_client.py +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse - -from fate_flow.controller.engine_adapt import build_engine -from fate_flow.operation.job_saver import JobSaver - - -FUNC = ["query_status", "download_log", "download_model"] - - -def call_fun(func, args): - job_id = args.job_id - role = args.role - party_id = args.party_id - component_name = args.component_name - output_path = args.output_path - engine, task = load_engine(job_id, role, party_id, component_name) - if func == "query_status": - query_status(engine, task) - elif func == "download_log": - download_log(engine, task, output_path) - - elif func == "download_model": - download_model(engine, task, output_path) - - -def load_engine(job_id, role, party_id, component_name): - tasks = JobSaver.query_task(job_id=job_id, role=role, party_id=party_id, component_name=component_name, run_on_this_party=True) - if tasks: - task = tasks[0] - if task.f_is_deepspeed: - deepspeed_engine = build_engine(task.f_engine_conf.get("computing_engine"), task.f_is_deepspeed) - return deepspeed_engine, task - else: - raise Exception(f"Not is a deepspeed task: job_id[{job_id}], role[{role}], party_id[{party_id}], component_name[{component_name}]") - else: - raise Exception(f"no found task: job_id[{job_id}], role[{role}], party_id[{party_id}], component_name[{component_name}]") - - -def query_status(engine, task): - status = engine._query_status(task) - print(status) - - -def download_log(engine, task, output_path): - engine.download_log(task, path=output_path) - print(output_path) - - -def download_model(engine, task, output_path): - engine.download_model(task, path=output_path) - print(output_path) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('-f', '--function', type=str, - choices=FUNC, - required=True, - help="function to call") - parser.add_argument('-j', '--job_id', required=True, type=str, help="job id") - parser.add_argument('-r', '--role', required=True, type=str, help="role") - parser.add_argument('-p', '--party_id', required=True, type=str, help="party id") - parser.add_argument('-cpn', '--component_name', required=True, type=str, help="component name") - parser.add_argument('-o', '--output_path', required=False, type=str, help="output_path") - args = parser.parse_args() - config_data = {} - config_data.update(dict((k, v) for k, v in vars(args).items() if v is not None)) - - call_fun(args.function, args) diff --git a/python/fate_flow/engine/__init__.py b/python/fate_flow/engine/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/engine/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/engine/backend/__init__.py b/python/fate_flow/engine/backend/__init__.py new file mode 100644 index 000000000..90057cf46 --- /dev/null +++ b/python/fate_flow/engine/backend/__init__.py @@ -0,0 +1,21 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_flow.engine.backend._session import build_backend + +__all__ = [ + "build_backend", +] + + diff --git a/python/fate_flow/engine/backend/_base.py b/python/fate_flow/engine/backend/_base.py new file mode 100644 index 000000000..60a0c4ec9 --- /dev/null +++ b/python/fate_flow/engine/backend/_base.py @@ -0,0 +1,150 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import abc +import logging +import os +import sys +import typing + +import yaml + +from fate_flow.db.db_models import Task +from fate_flow.entity.types import ProviderName, WorkerName +from fate_flow.manager.service.worker_manager import WorkerManager +from fate_flow.utils.job_utils import get_task_directory + + +class EngineABC(metaclass=abc.ABCMeta): + @abc.abstractmethod + def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_dir, cwd_dir, **kwargs) -> typing.Dict: + ... + + @abc.abstractmethod + def kill(self, task: Task): + ... + + @abc.abstractmethod + def is_alive(self, task: Task): + ... + + +class LocalEngine(object): + @classmethod + def get_component_define(cls, provider_name, task_info, stage): + task_dir = get_task_directory(**task_info, output=True) + component_ref = task_info.get("component") + role = task_info.get("role") + os.makedirs(task_dir, exist_ok=True) + define_file = os.path.join(task_dir, "define.yaml") + cmd = cls.generate_component_define_cmd(provider_name, component_ref, role, stage, define_file) + logging.debug(f"load define cmd: {cmd}") + if cmd: + WorkerManager.start_task_worker( + worker_name=WorkerName.COMPONENT_DEFINE, + task_info=task_info, + common_cmd=cmd, + sync=True + ) + if os.path.exists(define_file): + with open(define_file, "r") as fr: + return yaml.safe_load(fr) + return {} + + def _cleanup1(self, **kwargs): + # backend cleanup + pass + + def _cleanup2(self, provider_name, task_info, config, **kwargs): + # engine cleanup: computing、federation .. + cmd = self.generate_cleanup_cmd(provider_name) + + if cmd: + logging.info(f"start clean task, config: {config}") + WorkerManager.start_task_worker( + worker_name=WorkerName.TASK_EXECUTE_CLEAN, + task_info=task_info, + common_cmd=cmd, + task_parameters=config, + sync=True + ) + logging.info(f"clean success") + + def cleanup(self, provider_name, task_info, config, party_task_id, **kwargs): + self._cleanup1(session_id=party_task_id, task_info=task_info) + self._cleanup2(provider_name, task_info, config, **kwargs) + + @staticmethod + def generate_component_run_cmd(provider_name, conf_path, output_path=""): + if provider_name == ProviderName.FATE: + from fate_flow.manager.worker.fate_executor import FateSubmit + module_file_path = sys.modules[FateSubmit.__module__].__file__ + + elif provider_name == ProviderName.FATE_FLOW: + from fate_flow.manager.worker.fate_flow_executor import FateFlowSubmit + module_file_path = sys.modules[FateFlowSubmit.__module__].__file__ + + else: + raise ValueError(f"load provider {provider_name} failed") + os.environ.pop("FATE_TASK_CONFIG", None) + common_cmd = [ + module_file_path, + "component", + "execute", + "--config", + conf_path, + "--execution-final-meta-path", + output_path + ] + + return common_cmd + + @staticmethod + def generate_component_define_cmd(provider_name, component_ref, role, stage, define_file): + cmd = [] + if provider_name == ProviderName.FATE: + from fate_flow.manager.worker.fate_executor import FateSubmit + module_file_path = sys.modules[FateSubmit.__module__].__file__ + cmd = [ + module_file_path, + "component", + "artifact-type", + "--name", + component_ref, + "--role", + role, + "--stage", + stage, + "--output-path", + define_file + ] + return cmd + + @staticmethod + def generate_cleanup_cmd(provider_name): + cmd = [] + if provider_name == ProviderName.FATE: + from fate_flow.manager.worker.fate_executor import FateSubmit + module_file_path = sys.modules[FateSubmit.__module__].__file__ + cmd = [ + module_file_path, + "component", + "cleanup", + "--env-name", + "FATE_TASK_CONFIG", + ] + return cmd diff --git a/python/fate_flow/engine/backend/_eggroll.py b/python/fate_flow/engine/backend/_eggroll.py new file mode 100644 index 000000000..66dbd415d --- /dev/null +++ b/python/fate_flow/engine/backend/_eggroll.py @@ -0,0 +1,42 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import yaml + +from fate_flow.engine.backend._base import LocalEngine +from fate_flow.entity.spec.dag import TaskConfigSpec +from fate_flow.entity.types import WorkerName, ComputingEngine +from fate_flow.manager.service.worker_manager import WorkerManager + + +class EggrollEngine(LocalEngine): + def run(self, task_info, run_parameters, engine_run, provider_name, output_path, conf_path, sync=False, **kwargs): + parameters = TaskConfigSpec.parse_obj(run_parameters) + if parameters.conf.computing.type == ComputingEngine.EGGROLL: + # update eggroll options + cores = engine_run.pop("task_cores_per_node", None) + if cores: + engine_run["eggroll.session.processors.per.node"] = cores + parameters.conf.computing.metadata.options.update(engine_run) + with open(conf_path, "w") as f: + # update parameters + yaml.dump(parameters.dict(), f) + return WorkerManager.start_task_worker( + worker_name=WorkerName.TASK_EXECUTE, + task_info=task_info, + common_cmd=self.generate_component_run_cmd(provider_name, conf_path, output_path, ), + sync=sync, + **kwargs + ).returncode diff --git a/python/fate_flow/engine/backend/_eggroll_deepspeed.py b/python/fate_flow/engine/backend/_eggroll_deepspeed.py new file mode 100644 index 000000000..f5d75fad9 --- /dev/null +++ b/python/fate_flow/engine/backend/_eggroll_deepspeed.py @@ -0,0 +1,195 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import logging +import os.path +import sys +import time +import traceback + +from fate_flow.engine.backend._base import LocalEngine +from fate_flow.engine.devices.deepspeed import EggrollDeepspeedEngine +from fate_flow.entity.spec.dag import TaskConfigSpec, ComponentOutputMeta, ArtifactOutputSpec +from fate_flow.entity.types import BaseStatus, TaskStatus, ComputingEngine +from fate_flow.manager.outputs.data import DataManager +from fate_flow.manager.worker.fate_ds_executor import FateSubmit +from fate_flow.runtime.system_settings import COMPUTING_CONF, DEEPSPEED_RESULT_PLACEHOLDER, MODEL_STORE_PATH, \ + DEEPSPEED_LOGS_DIR_PLACEHOLDER, DEEPSPEED_MODEL_DIR_PLACEHOLDER +from fate_flow.utils.job_utils import generate_deepspeed_id + +logger = logging.getLogger(__name__) + + +class StatusSet(BaseStatus): + NEW = "NEW" + NEW_TIMEOUT = "NEW_TIMEOUT" + ACTIVE = "ACTIVE" + CLOSED = "CLOSED" + KILLED = "KILLED" + ERROR = "ERROR" + FINISHED = "FINISHED" + + +class EndStatus(BaseStatus): + NEW_TIMEOUT = StatusSet.NEW_TIMEOUT + CLOSED = StatusSet.CLOSED + FAILED = StatusSet.KILLED + ERROR = StatusSet.ERROR + FINISHED = StatusSet.FINISHED + + +class Deepspeed(LocalEngine): + def run(self, output_path, engine_run, run_parameters, session_id, task_info, **kwargs): + parameters = TaskConfigSpec.parse_obj(run_parameters) + env_name = "FATE_TASK_CONFIG" + self.start_submit(session_id, parameters, engine_run, env_name) + status = self.wait_deepspeed_job(session_id=session_id, timeout=engine_run.get("timeout", 36000)) + logger.info(f"deepspeed task end with status {status}") + engine = EggrollDeepspeedEngine() + if status not in EndStatus.status_list(): + logger.info(f"start to kill deepspeed {session_id} task") + self.kill(session_id=session_id) + return -1 + logger.info(f"start download task result to dir {os.path.dirname(output_path)}") + engine.download_result( + worker_id=generate_deepspeed_id(parameters.party_task_id), + path=os.path.dirname(output_path) + ) + logger.info(f"start download task model") + output_meta = None + logger.info(f"{output_path}: {os.path.exists(output_path)}") + if os.path.exists(output_path): + with open(output_path, "r") as f: + try: + result = json.load(f) + output_meta = ComponentOutputMeta.parse_obj(result) + except: + logger.info(f"load output path {output_path} failed") + logger.info(output_meta) + if output_meta: + if output_meta.status.code != 0: + raise RuntimeError(output_meta.dict()) + for _key, _model in output_meta.io_meta.outputs.model.items(): + model = ArtifactOutputSpec(**_model) + _, address = DataManager.uri_to_address(model.uri) + path = os.path.join(MODEL_STORE_PATH, address.path.split("models/")[-1]) + logger.info(f"download model to {path}") + engine.download_model_do(worker_id=session_id, path=path) + logger.info("download model success") + return 0 + + @classmethod + def start_submit(cls, session_id, parameters: TaskConfigSpec, engine_run, env_name): + from eggroll.deepspeed.submit import client + host = COMPUTING_CONF.get(ComputingEngine.EGGROLL).get("host") + port = COMPUTING_CONF.get(ComputingEngine.EGGROLL).get("port") + client = client.DeepspeedJob(session_id=session_id, host=host, port=port) + world_size = engine_run.get("cores", 1) + timeout_seconds = engine_run.get("timeout_seconds", 21600) + resource_exhausted_strategy = engine_run.get("resource_exhausted_strategy", "waiting") + options = { + "eggroll.container.deepspeed.script.path": sys.modules[FateSubmit.__module__].__file__ + } + resource_options = {"timeout_seconds": timeout_seconds, "resource_exhausted_strategy": resource_exhausted_strategy} + resource_options.update(engine_run) + command_arguments = cls.generate_command_arguments(env_name) + environment_variables = { + env_name: json.dumps(parameters.dict()), + "DEEPSPEED_LOGS_DIR_PLACEHOLDER": DEEPSPEED_LOGS_DIR_PLACEHOLDER, + "DEEPSPEED_MODEL_DIR_PLACEHOLDER": DEEPSPEED_MODEL_DIR_PLACEHOLDER, + "DEEPSPEED_RESULT_PLACEHOLDER": DEEPSPEED_RESULT_PLACEHOLDER + } + logger.info(f"world size {world_size}") + logger.info(f"command_arguments: {command_arguments}") + logger.info(f"environment_variables: {environment_variables}") + logger.info(f"resource_options: {resource_options}") + logger.info(f"options: {options}") + logger.info(f"start submit deepspeed task {session_id}") + client.submit( + world_size=world_size, + command_arguments=command_arguments, + environment_variables=environment_variables, + files={}, + resource_options=resource_options, + options=options + ) + logger.info(f"submit deepspeed task success") + + def wait_deepspeed_job(self, session_id, timeout=36000): + if timeout < 0: + return + + while True: + status = self._query_status(session_id=session_id) + if timeout % 5 == 0: + logger.info(f"task status: {status}") + timeout -= 1 + if timeout == 0: + logger.error(f"task timeout, total {timeout}s") + return status + elif status in EndStatus.status_list(): + return status + time.sleep(1) + + @staticmethod + def generate_command_arguments(env_name, output_path=f"{DEEPSPEED_RESULT_PLACEHOLDER}/task_result.yaml"): + command_arguments = [ + "component", + "execute", + "--env-name", + env_name, + "--execution-final-meta-path", + output_path + ] + return command_arguments + + def _cleanup1(self, session_id, task_info, **kwargs): + # self.kill(session_id) + pass + + @staticmethod + def kill(session_id): + if session_id: + logger.info(f"start kill deepspeed task {session_id}") + from eggroll.deepspeed.submit import client + host = COMPUTING_CONF.get(ComputingEngine.EGGROLL).get("host") + port = COMPUTING_CONF.get(ComputingEngine.EGGROLL).get("port") + client = client.DeepspeedJob(session_id, host=host, port=port) + try: + client.kill() + except Exception as e: + traceback.format_exc() + logger.error(e) + + @staticmethod + def _query_status(session_id): + if session_id: + from eggroll.deepspeed.submit import client + host = COMPUTING_CONF.get(ComputingEngine.EGGROLL).get("host") + port = COMPUTING_CONF.get(ComputingEngine.EGGROLL).get("port") + client = client.DeepspeedJob(session_id, host=host, port=port) + _s = client.query_status().status + return _s if _s else StatusSet.NEW + return StatusSet.NEW + + def query_status(self, session_id): + status = self._query_status(session_id) + if status in EndStatus.status_list(): + if status in [EndStatus.FINISHED]: + return TaskStatus.SUCCESS + else: + return TaskStatus.FAILED + return TaskStatus.RUNNING diff --git a/python/fate_flow/engine/backend/_session.py b/python/fate_flow/engine/backend/_session.py new file mode 100644 index 000000000..18c4ac3e9 --- /dev/null +++ b/python/fate_flow/engine/backend/_session.py @@ -0,0 +1,34 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.engine.backend._eggroll import EggrollEngine +from fate_flow.engine.backend._spark import SparkEngine +from fate_flow.engine.backend._eggroll_deepspeed import Deepspeed +from fate_flow.entity.types import ComputingEngine, LauncherType + + +def build_backend(backend_name: str, launcher_name: str = LauncherType.DEFAULT): + if backend_name in {ComputingEngine.EGGROLL, ComputingEngine.STANDALONE}: + if launcher_name == LauncherType.DEEPSPEED: + backend = Deepspeed() + elif not launcher_name or launcher_name == LauncherType.DEFAULT: + backend = EggrollEngine() + else: + raise ValueError(f'backend "{backend_name}" launcher {launcher_name} is not supported') + elif backend_name == ComputingEngine.SPARK: + backend = SparkEngine() + else: + raise ValueError(f'backend "{backend_name}" is not supported') + return backend diff --git a/python/fate_flow/engine/backend/_spark.py b/python/fate_flow/engine/backend/_spark.py new file mode 100644 index 000000000..3f7782720 --- /dev/null +++ b/python/fate_flow/engine/backend/_spark.py @@ -0,0 +1,56 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +from fate_flow.engine.backend._base import LocalEngine +from fate_flow.entity.types import WorkerName +from fate_flow.manager.service.worker_manager import WorkerManager + + +class SparkEngine(LocalEngine): + def run(self, task_info, run_parameters, conf_path, output_path, engine_run, provider_name, **kwargs): + spark_home = os.environ.get("SPARK_HOME", None) + if not spark_home: + try: + import pyspark + spark_home = pyspark.__path__[0] + except ImportError as e: + raise RuntimeError("can not import pyspark") + except Exception as e: + raise RuntimeError("can not import pyspark") + + deploy_mode = engine_run.get("deploy-mode", "client") + if deploy_mode not in ["client"]: + raise ValueError(f"deploy mode {deploy_mode} not supported") + + spark_submit_cmd = os.path.join(spark_home, "bin/spark-submit") + process_cmd = [spark_submit_cmd, f"--name={task_info.get('task_id')}#{task_info.get('role')}"] + for k, v in engine_run.items(): + if k != "conf": + process_cmd.append(f"--{k}={v}") + if "conf" in engine_run: + for ck, cv in engine_run["conf"].items(): + process_cmd.append(f"--conf") + process_cmd.append(f"{ck}={cv}") + extra_env = {"SPARK_HOME": spark_home} + return WorkerManager.start_task_worker( + worker_name=WorkerName.TASK_EXECUTE, + task_info=task_info, + common_cmd=self.generate_component_run_cmd(provider_name, conf_path, output_path), + extra_env=extra_env, + executable=process_cmd, + sync=True + ).returncode diff --git a/python/fate_flow/engine/devices/__init__.py b/python/fate_flow/engine/devices/__init__.py new file mode 100644 index 000000000..dede5eda4 --- /dev/null +++ b/python/fate_flow/engine/devices/__init__.py @@ -0,0 +1,42 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_flow.engine.devices._base import EngineABC +from fate_flow.entity.types import ProviderDevice, LauncherType +from fate_flow.manager.service.provider_manager import ProviderManager + + +def build_engine(provider_name: str, launcher_name: str = LauncherType.DEFAULT): + provider = ProviderManager.get_provider_by_provider_name(provider_name) + + if launcher_name == LauncherType.DEEPSPEED: + if provider.device in {ProviderDevice.LOCAL}: + from fate_flow.engine.devices.deepspeed import EggrollDeepspeedEngine + engine_session = EggrollDeepspeedEngine(provider) + else: + raise ValueError(f'engine launcher {LauncherType.DEEPSPEED} device "{provider.device}" is not supported') + + else: + if provider.device in {ProviderDevice.DOCKER, ProviderDevice.K8S}: + from fate_flow.engine.devices.container import ContainerdEngine + engine_session = ContainerdEngine(provider) + + elif provider.device in {ProviderDevice.LOCAL}: + from fate_flow.engine.devices.local import LocalEngine + engine_session = LocalEngine(provider) + + else: + raise ValueError(f'engine device "{provider.device}" is not supported') + + return engine_session diff --git a/python/fate_flow/controller/engine_controller/engine.py b/python/fate_flow/engine/devices/_base.py similarity index 87% rename from python/fate_flow/controller/engine_controller/engine.py rename to python/fate_flow/engine/devices/_base.py index dc781c9ac..83febff02 100644 --- a/python/fate_flow/controller/engine_controller/engine.py +++ b/python/fate_flow/engine/devices/_base.py @@ -33,3 +33,11 @@ def kill(self, task: Task): @abc.abstractmethod def is_alive(self, task: Task): ... + + @abc.abstractmethod + def cleanup(self, task: Task): + ... + + @abc.abstractmethod + def download_output(self, task: Task): + ... diff --git a/python/fate_flow/engine/devices/container.py b/python/fate_flow/engine/devices/container.py new file mode 100644 index 000000000..6bbd45894 --- /dev/null +++ b/python/fate_flow/engine/devices/container.py @@ -0,0 +1,85 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json + +import yaml + +from fate_flow.db.db_models import Task +from fate_flow.engine.devices._base import EngineABC +from fate_flow.entity.types import ProviderDevice +from fate_flow.runtime.component_provider import ComponentProvider +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.utils.log_utils import schedule_logger + + +class ContainerdEngine(EngineABC): + def __init__(self, provider: ComponentProvider): + + if provider.device == ProviderDevice.K8S: + from fate_flow.manager.container.k8s_manager import K8sManager + self.manager = K8sManager(provider) + + elif provider.device == ProviderDevice.DOCKER: + from fate_flow.manager.container.docker_manager import DockerManager + self.manager = DockerManager(provider) + + else: + raise ValueError(f'worker "{provider.device}" is not supported') + + @staticmethod + def _get_name(task: Task): + return f'{task.f_role}-{task.f_party_id}-{task.f_task_id}-{task.f_task_version}' + + @classmethod + def _flatten_dict(cls, data, parent_key='', sep='.'): + items = {} + for key, value in data.items(): + new_key = f"{parent_key}{sep}{key}" if parent_key else key + if isinstance(value, dict): + items.update(cls._flatten_dict(value, new_key, sep=sep)) + else: + items[new_key] = value + return items + + @classmethod + def _get_environment(cls, task: Task, run_parameters): + return cls._flatten_dict(run_parameters) + + @classmethod + def _get_volume(cls, task): + return None + + def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_dir, cwd_dir, **kwargs): + name = self._get_name(task) + cmd = None + env = self._get_environment(task, run_parameters) + schedule_logger(job_id=task.f_job_id).info(f"start run container {name}, cmd: {cmd}, env: {json.dumps(env)}") + self.manager.start(name, cmd, env, volumes=self._get_volume(task)) + return { + 'run_ip': RuntimeConfig.JOB_SERVER_HOST + } + + def kill(self, task: Task): + self.manager.stop(self._get_name(task)) + + def is_alive(self, task: Task): + return self.manager.is_running(self._get_name(task)) + + def cleanup(self, task: Task): + pass + + def download_output(self, task: Task): + pass diff --git a/python/fate_flow/engine/devices/deepspeed.py b/python/fate_flow/engine/devices/deepspeed.py new file mode 100644 index 000000000..7191e453f --- /dev/null +++ b/python/fate_flow/engine/devices/deepspeed.py @@ -0,0 +1,229 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import datetime +import logging +import os +import sys +import traceback + +from fate_flow.db import Task +from fate_flow.engine.devices.local import LocalEngine +from fate_flow.entity.types import BaseStatus, TaskStatus, WorkerName, StorageEngine +from fate_flow.manager.service.worker_manager import WorkerManager +from fate_flow.manager.worker.deepspeed_download_model import DownloadModel +from fate_flow.runtime.system_settings import MODEL_STORE_PATH, COMPUTING_CONF +from fate_flow.utils import job_utils, process_utils +from fate_flow.utils.job_utils import get_job_log_directory, generate_deepspeed_id +from fate_flow.utils.log_utils import schedule_logger + +logger = logging.getLogger(__name__) + + +class StatusSet(BaseStatus): + NEW = "NEW" + NEW_TIMEOUT = "NEW_TIMEOUT" + ACTIVE = "ACTIVE" + CLOSED = "CLOSED" + KILLED = "KILLED" + ERROR = "ERROR" + FINISHED = "FINISHED" + + +class EndStatus(BaseStatus): + NEW_TIMEOUT = StatusSet.NEW_TIMEOUT + CLOSED = StatusSet.CLOSED + FAILED = StatusSet.KILLED + ERROR = StatusSet.ERROR + FINISHED = StatusSet.FINISHED + + +class EggrollDeepspeedEngine(LocalEngine): + @staticmethod + def generate_session_id(): + return f"deepspeed_session_{datetime.datetime.now().strftime('%Y%m%d-%H%M%S-%f')}" + + def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_dir, cwd_dir, **kwargs): + schedule_logger(task.f_job_id).info("start to submit deepspeed task") + run_info = WorkerManager.start_task_worker( + worker_name=WorkerName.TASK_ENTRYPOINT, + task_info=task.to_human_model_dict(), + extra_env={"PYTHONPATH": self.provider.python_path}, + executable=[self.provider.python_env], + common_cmd=self.generate_cmd(), + task_parameters=run_parameters, + record=True + ) + run_info["worker_id"] = generate_deepspeed_id(run_parameters.get("party_task_id")) + return run_info + + def cleanup(self, task: Task): + self._cleanup(task, sync=True) + self.kill(task) + + def kill(self, task): + schedule_logger(task.f_job_id).info(f"start kill deepspeed task {task.f_worker_id}") + from eggroll.deepspeed.submit import client + + host = COMPUTING_CONF.get(StorageEngine.EGGROLL).get("host") + port = COMPUTING_CONF.get(StorageEngine.EGGROLL).get("port") + + client = client.DeepspeedJob(task.f_worker_id, host=host, port=port) + try: + client.kill() + except Exception as e: + traceback.format_exc() + schedule_logger(task.f_job_id).error(e) + + def download_output(self, task: Task): + try: + schedule_logger(task.f_job_id).info(f"start download logs") + self.download_log(task) + except Exception as e: + traceback.format_exc() + schedule_logger(task.f_job_id).error(e) + + @staticmethod + def _query_status(session_id): + if session_id: + from eggroll.deepspeed.submit import client + host = COMPUTING_CONF.get(StorageEngine.EGGROLL).get("host") + port = COMPUTING_CONF.get(StorageEngine.EGGROLL).get("port") + client = client.DeepspeedJob(session_id, host=host, port=port) + _s = client.query_status().status + return _s if _s else StatusSet.NEW + return StatusSet.NEW + + def query_task_status(self, task): + status = self._query_status(task.f_worker_id) + if status in EndStatus.status_list(): + if status in [EndStatus.FINISHED]: + return TaskStatus.SUCCESS + else: + return TaskStatus.FAILED + + @staticmethod + def _download_job(session_id, base_dir, content_type=None, ranks: list = None): + from eggroll.deepspeed.submit import client + if not content_type: + content_type = client.ContentType.ALL + if session_id: + host = COMPUTING_CONF.get(StorageEngine.EGGROLL).get("host") + port = COMPUTING_CONF.get(StorageEngine.EGGROLL).get("port") + client = client.DeepspeedJob(session_id, host=host, port=port) + os.makedirs(base_dir, exist_ok=True) + path = lambda rank: f"{base_dir}/{rank}.zip" + client.download_job_to(rank_to_path=path, content_type=content_type, ranks=ranks) + return base_dir + + def query_status(self, session_id): + status = self._query_status(session_id) + if status in EndStatus.status_list(): + if status in [EndStatus.FINISHED]: + return TaskStatus.SUCCESS + else: + return TaskStatus.FAILED + + def is_alive(self, task): + status = self._query_status(task.f_worker_id) + if status in StatusSet.status_list(): + if status in EndStatus.status_list(): + return False + else: + return True + else: + raise RuntimeError(f"task run status: {status}") + + def download(self, base_dir, content_type=None, ranks=None, worker_id=None, only_rank_0=False): + from eggroll.deepspeed.submit.client import ContentType + if not content_type: + content_type = ContentType.ALL + session_id = worker_id + dir_name = self._download_job(session_id, base_dir, content_type, ranks) + if dir_name: + for file in os.listdir(dir_name): + if file.endswith(".zip"): + if only_rank_0: + rank_dir = dir_name + else: + rank_dir = os.path.join(dir_name, file.split(".zip")[0]) + os.makedirs(rank_dir, exist_ok=True) + self.unzip(os.path.join(dir_name, file), extra_dir=rank_dir) + os.remove(os.path.join(dir_name, file)) + + def download_log(self, task, path=None): + from eggroll.deepspeed.submit.client import ContentType + if not path: + path = self.log_path(task) + schedule_logger(task.f_job_id).info(f"download logs to {path}") + self.download(worker_id=task.f_worker_id, base_dir=path, content_type=ContentType.LOGS) + + def download_result(self, path, worker_id): + from eggroll.deepspeed.submit.client import ContentType + self.download( + worker_id=worker_id, base_dir=path, content_type=ContentType.RESULT, ranks=[0], + only_rank_0=True + ) + + @staticmethod + def download_model(task_info, path=""): + # run subprocess to download model + process_cmd = [ + sys.executable or 'python3', + sys.modules[DownloadModel.__module__].__file__, + '--job_id', task_info.get("job_id"), + '--role', task_info.get("role"), + '--party_id', task_info.get("party_id"), + '--task_id', task_info.get("task_id"), + '--task_version', task_info.get("task_version"), + "--path", path + ] + process_name = "model_download" + log_dir = conf_dir = job_utils.get_job_log_directory(task_info.get("job_id"), task_info.get("role"), task_info.get("party_id"), task_info.get("task_name")) + p = process_utils.run_subprocess( + job_id=task_info.get("job_id"), + config_dir=conf_dir, + process_cmd=process_cmd, + std_dir=log_dir, + process_name=process_name + ) + schedule_logger(task_info.get("job_id")).info(f"download model process id: {p.pid}") + + def download_model_do(self, task=None, worker_id=None, path=None): + from eggroll.deepspeed.submit.client import ContentType + if not path: + path = self.model_path(task) + self.download(worker_id=worker_id, base_dir=path, content_type=ContentType.MODELS, ranks=[0], only_rank_0=True) + + @staticmethod + def unzip(zip_path, extra_dir): + import zipfile + zfile = zipfile.ZipFile(zip_path, "r") + for name in zfile.namelist(): + dir_name = os.path.dirname(zip_path) + file_path = os.path.join(dir_name, extra_dir, name) + os.makedirs(os.path.dirname(file_path), exist_ok=True) + data = zfile.read(name) + with open(file_path, "w+b") as file: + file.write(data) + + @staticmethod + def model_path(task): + _p = os.path.join(MODEL_STORE_PATH, task.f_job_id, task.f_role, task.f_party_id, task.f_task_name) + return _p + + @staticmethod + def log_path(task): + return get_job_log_directory(task.f_job_id, task.f_role, task.f_party_id, task.f_task_name) diff --git a/python/fate_flow/engine/devices/local.py b/python/fate_flow/engine/devices/local.py new file mode 100644 index 000000000..0d3c31d4a --- /dev/null +++ b/python/fate_flow/engine/devices/local.py @@ -0,0 +1,87 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import sys + +from fate_flow.db.db_models import Task +from fate_flow.engine.devices._base import EngineABC +from fate_flow.entity.types import WorkerName +from fate_flow.manager.service.worker_manager import WorkerManager +from fate_flow.manager.worker.fate_flow_executor import FateFlowSubmit +from fate_flow.runtime.component_provider import ComponentProvider +from fate_flow.utils import process_utils + + +class LocalEngine(EngineABC): + def __init__(self, provider: ComponentProvider = None): + self.provider = provider + + def run(self, task: Task, run_parameters, run_parameters_path, config_dir, log_dir, cwd_dir, **kwargs): + return WorkerManager.start_task_worker( + worker_name=WorkerName.TASK_ENTRYPOINT, + task_info=task.to_human_model_dict(), + extra_env={"PYTHONPATH": self.provider.python_path}, + executable=[self.provider.python_env], + common_cmd=self.generate_cmd(), + task_parameters=run_parameters, + record=True + ) + + def kill(self, task): + process_utils.kill_task_executor_process(task) + + def is_alive(self, task): + return process_utils.check_process(pid=int(task.f_run_pid), task=task) + + def cleanup(self, task: Task): + return self._cleanup(task) + + def _cleanup(self, task: Task, sync=False): + return WorkerManager.start_task_worker( + worker_name=WorkerName.TASK_CLEAN, + task_info=task.to_human_model_dict(), + extra_env={"PYTHONPATH": self.provider.python_path}, + executable=[self.provider.python_env], + common_cmd=self.generate_cleanup_cmd(), + task_parameters=task.f_component_parameters, + sync=sync + ) + + def download_output(self, task): + pass + + @staticmethod + def generate_cmd(): + module_file_path = sys.modules[FateFlowSubmit.__module__].__file__ + common_cmd = [ + module_file_path, + "component", + "entrypoint", + "--env-name", + "FATE_TASK_CONFIG", + ] + return common_cmd + + @staticmethod + def generate_cleanup_cmd(): + module_file_path = sys.modules[FateFlowSubmit.__module__].__file__ + common_cmd = [ + module_file_path, + "component", + "cleanup", + "--env-name", + "FATE_TASK_CONFIG", + ] + return common_cmd diff --git a/python/fate_flow/engine/relation_ship.py b/python/fate_flow/engine/relation_ship.py new file mode 100644 index 000000000..3693f0548 --- /dev/null +++ b/python/fate_flow/engine/relation_ship.py @@ -0,0 +1,80 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.entity.types import StandaloneAddress, EggRollAddress, HDFSAddress, MysqlAddress, HiveAddress, \ + PathAddress, ApiAddress, ComputingEngine, StorageEngine, FederationEngine, EngineType, FileAddress + + +class Relationship(object): + Computing = { + ComputingEngine.STANDALONE: { + EngineType.STORAGE: { + "default": StorageEngine.STANDALONE, + "support": [StorageEngine.STANDALONE], + }, + EngineType.FEDERATION: { + "default": FederationEngine.STANDALONE, + "support": [ + FederationEngine.STANDALONE, + FederationEngine.RABBITMQ, + FederationEngine.PULSAR, + FederationEngine.OSX, + FederationEngine.ROLLSITE + ], + }, + }, + ComputingEngine.EGGROLL: { + EngineType.STORAGE: { + "default": StorageEngine.EGGROLL, + "support": [StorageEngine.EGGROLL], + }, + EngineType.FEDERATION: { + "default": FederationEngine.ROLLSITE, + "support": [ + FederationEngine.ROLLSITE, + FederationEngine.RABBITMQ, + FederationEngine.PULSAR, + FederationEngine.OSX + ], + }, + }, + ComputingEngine.SPARK: { + EngineType.STORAGE: { + "default": StorageEngine.HDFS, + "support": [ + StorageEngine.HDFS, + StorageEngine.HIVE, + StorageEngine.FILE, + StorageEngine.STANDALONE + ], + }, + EngineType.FEDERATION: { + "default": FederationEngine.RABBITMQ, + "support": [FederationEngine.PULSAR, FederationEngine.RABBITMQ, FederationEngine.OSX, FederationEngine.STANDALONE], + }, + } + } + + EngineToAddress = { + StorageEngine.STANDALONE: StandaloneAddress, + StorageEngine.EGGROLL: EggRollAddress, + StorageEngine.HDFS: HDFSAddress, + StorageEngine.MYSQL: MysqlAddress, + StorageEngine.HIVE: HiveAddress, + StorageEngine.FILE: FileAddress, + StorageEngine.PATH: PathAddress, + StorageEngine.API: ApiAddress + } + diff --git a/python/fate_flow/engine/storage/__init__.py b/python/fate_flow/engine/storage/__init__.py new file mode 100644 index 000000000..f2e4f13dd --- /dev/null +++ b/python/fate_flow/engine/storage/__init__.py @@ -0,0 +1,17 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_flow.engine.storage._types import EggRollStoreType, StorageEngine, StandaloneStoreType, DataType, StorageOrigin +from fate_flow.engine.storage._table import StorageTableBase, StorageTableMeta +from fate_flow.engine.storage._session import StorageSessionBase, Session diff --git a/python/fate_flow/engine/storage/_abc.py b/python/fate_flow/engine/storage/_abc.py new file mode 100644 index 000000000..854bef149 --- /dev/null +++ b/python/fate_flow/engine/storage/_abc.py @@ -0,0 +1,242 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import abc +from typing import Iterable + + +class StorageTableMetaABC(metaclass=abc.ABCMeta): + @abc.abstractmethod + def create(self): + ... + + @abc.abstractmethod + def set_metas(self, **kwargs): + ... + + @abc.abstractmethod + def query_table_meta(self, filter_fields, query_fields=None): + ... + + @abc.abstractmethod + def update_metas(self, data_meta=None, count=None, part_of_data=None, description=None, partitions=None, **kwargs): + ... + + @abc.abstractmethod + def destroy_metas(self): + ... + + @abc.abstractmethod + def get_name(self): + ... + ... + + @abc.abstractmethod + def get_namespace(self): + ... + + @abc.abstractmethod + def get_address(self): + ... + + @abc.abstractmethod + def get_engine(self): + ... + + @abc.abstractmethod + def get_store_type(self): + ... + + @abc.abstractmethod + def get_options(self): + ... + + @abc.abstractmethod + def get_partitions(self): + ... + + @abc.abstractmethod + def get_id_delimiter(self): + ... + + @abc.abstractmethod + def get_extend_sid(self): + ... + + @abc.abstractmethod + def get_auto_increasing_sid(self): + ... + + @abc.abstractmethod + def get_have_head(self): + ... + + @abc.abstractmethod + def get_data_meta(self): + ... + + @abc.abstractmethod + def get_count(self): + ... + + @abc.abstractmethod + def get_part_of_data(self): + ... + + @abc.abstractmethod + def get_description(self): + ... + + @abc.abstractmethod + def get_source(self): + ... + + @abc.abstractmethod + def get_disable(self): + ... + + @abc.abstractmethod + def to_dict(self) -> dict: + ... + + +class StorageTableABC(metaclass=abc.ABCMeta): + @property + @abc.abstractmethod + def name(self): + ... + + @property + @abc.abstractmethod + def namespace(self): + ... + + @property + @abc.abstractmethod + def address(self): + ... + + @property + @abc.abstractmethod + def engine(self): + ... + + @property + @abc.abstractmethod + def options(self): + ... + + @property + @abc.abstractmethod + def partitions(self): + ... + + @property + @abc.abstractmethod + def data_type(self): + ... + + @property + @abc.abstractmethod + def meta(self) -> StorageTableMetaABC: + ... + + @meta.setter + @abc.abstractmethod + def meta(self, meta: StorageTableMetaABC): + ... + + @abc.abstractmethod + def update_meta(self, + schema=None, + count=None, + part_of_data=None, + description=None, + partitions=None, + **kwargs) -> StorageTableMetaABC: + ... + + @abc.abstractmethod + def create_meta(self, **kwargs) -> StorageTableMetaABC: + ... + + @abc.abstractmethod + def put_all(self, kv_list: Iterable, **kwargs): + ... + + @abc.abstractmethod + def collect(self, **kwargs) -> list: + ... + + @abc.abstractmethod + def read(self) -> list: + ... + + @abc.abstractmethod + def count(self): + ... + + @abc.abstractmethod + def destroy(self): + ... + + @abc.abstractmethod + def check_address(self): + ... + + +class StorageSessionABC(metaclass=abc.ABCMeta): + @abc.abstractmethod + def create_table(self, address, name, namespace, partitions, key_serdes_type, value_serdes_type, partitioner_type, + storage_type=None, options=None, + **kwargs) -> StorageTableABC: + ... + + @abc.abstractmethod + def get_table(self, name, namespace) -> StorageTableABC: + ... + + @abc.abstractmethod + def get_table_meta(self, name, namespace) -> StorageTableMetaABC: + ... + + # @abc.abstractmethod + # def table(self, name, namespace, address, partitions, store_type=None, options=None, **kwargs) -> StorageTableABC: + # ... + + # @abc.abstractmethod + # def get_storage_info(self, name, namespace): + # ... + + @abc.abstractmethod + def destroy(self): + ... + + @abc.abstractmethod + def stop(self): + ... + + @abc.abstractmethod + def kill(self): + ... + + @property + @abc.abstractmethod + def session_id(self) -> str: + ... + + @property + @abc.abstractmethod + def engine(self) -> str: + ... diff --git a/python/fate_flow/engine/storage/_partitioner.py b/python/fate_flow/engine/storage/_partitioner.py new file mode 100644 index 000000000..ee4e18b9a --- /dev/null +++ b/python/fate_flow/engine/storage/_partitioner.py @@ -0,0 +1,73 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import hashlib + + +def partitioner(hash_func, total_partitions): + def partition(key): + return hash_func(key) % total_partitions + + return partition + + +def integer_partitioner(key: bytes, total_partitions): + return int.from_bytes(key, "big") % total_partitions + + +def mmh3_partitioner(key: bytes, total_partitions): + import mmh3 + + return mmh3.hash(key) % total_partitions + + +def _java_string_like_partitioner(key, total_partitions): + _key = hashlib.sha1(key).digest() + _key = int.from_bytes(_key, byteorder="little", signed=False) + b, j = -1, 0 + while j < total_partitions: + b = int(j) + _key = ((_key * 2862933555777941757) + 1) & 0xFFFFFFFFFFFFFFFF + j = float(b + 1) * (float(1 << 31) / float((_key >> 33) + 1)) + return int(b) + + +def get_default_partitioner(): + return mmh3_partitioner + # return _java_string_like_partitioner + + +def get_partitioner_by_type(partitioner_type: int): + if partitioner_type == 0: + return get_default_partitioner() + elif partitioner_type == 1: + return integer_partitioner + elif partitioner_type == 2: + return mmh3_partitioner + else: + raise ValueError(f"partitioner type `{partitioner_type}` not supported") + + +def create_partitioner(partitioner_type): + if partitioner_type is None: + return mmh3_partitioner + if partitioner_type == "integer": + return integer_partitioner + elif partitioner_type == "mmh3": + return mmh3_partitioner + else: + raise ValueError("invalid partitioner type: {}".format(partitioner_type)) diff --git a/python/fate_flow/engine/storage/_session.py b/python/fate_flow/engine/storage/_session.py new file mode 100644 index 000000000..b24ce84ec --- /dev/null +++ b/python/fate_flow/engine/storage/_session.py @@ -0,0 +1,366 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import typing +import uuid + +import peewee + +from fate_flow.db.base_models import DB +from fate_flow.db.storage_models import SessionRecord +from fate_flow.engine.storage._abc import ( + StorageSessionABC, + StorageTableABC, + StorageTableMetaABC, +) + +from fate_flow.engine.storage._table import StorageTableMeta +from fate_flow.entity.types import EngineType, StorageEngine +from fate_flow.runtime.system_settings import ENGINES, COMPUTING_CONF +from fate_flow.utils import base_utils +from fate_flow.utils.log import getLogger + +LOGGER = getLogger("storage") + + +class StorageSessionBase(StorageSessionABC): + def __init__(self, session_id, engine): + self._session_id = session_id + self._engine = engine + + def create_table( + self, + address, + name, + namespace, + partitions, + key_serdes_type=0, + value_serdes_type=0, + partitioner_type=0, + **kwargs, + ): + table = self.table( + address=address, + name=name, + namespace=namespace, + partitions=partitions, + key_serdes_type=key_serdes_type, + value_serdes_type=value_serdes_type, + partitioner_type=partitioner_type, + **kwargs, + ) + table.create_meta(**kwargs) + return table + + @staticmethod + def meta_table_name(name): + return f"{name}.meta" + + def get_table(self, name, namespace): + meta = StorageTableMeta(name=name, namespace=namespace) + if meta and meta.exists(): + table = self.load( + name=meta.get_name(), + namespace=meta.get_namespace(), + address=meta.get_address(), + partitions=meta.get_partitions(), + store_type=meta.get_store_type(), + options=meta.get_options(), + ) + table.meta = meta + return table + else: + return None + + @classmethod + def get_table_meta(cls, name, namespace): + meta = StorageTableMeta(name=name, namespace=namespace) + if meta and meta.exists(): + return meta + else: + return None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.destroy() + + def destroy(self): + try: + self.stop() + except Exception as e: + LOGGER.warning( + f"stop storage session {self._session_id} failed, try to kill", e + ) + self.kill() + + def table( + self, + name, + namespace, + address, + partitions, + key_serdes_type, + value_serdes_type, + partitioner_type, + store_type, + options, + **kwargs, + ): + raise NotImplementedError() + + def load( + self, + name, + namespace, + address, + store_type, + partitions, + options=None, + **kwargs, + ): + raise NotImplementedError() + + def stop(self): + raise NotImplementedError() + + def kill(self): + raise NotImplementedError() + + @property + def session_id(self): + return self._session_id + + @property + def engine(self): + return self._engine + + +class Session(object): + __GLOBAL_SESSION = None + + @classmethod + def get_global(cls): + return cls.__GLOBAL_SESSION + + @classmethod + def _as_global(cls, sess): + cls.__GLOBAL_SESSION = sess + + def as_global(self): + self._as_global(self) + return self + + def __init__(self, session_id: str = None, options=None): + if options is None: + options = {} + self._storage_engine = ENGINES.get(EngineType.STORAGE, None) + self._storage_session: typing.Dict[str, StorageSessionABC] = {} + self._session_id = str(uuid.uuid1()) if not session_id else session_id + self._logger = ( + LOGGER + if options.get("logger", None) is None + else options.get("logger", None) + ) + + self._logger.info(f"create manager session {self._session_id}") + + @property + def session_id(self) -> str: + return self._session_id + + def _open(self): + return self + + def _close(self): + self.destroy_all_sessions() + + def __enter__(self): + return self._open() + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_tb: + self._logger.exception("", exc_info=(exc_type, exc_val, exc_tb)) + return self._close() + + def _get_or_create_storage( + self, + storage_session_id=None, + storage_engine=None, + record: bool = True, + **kwargs, + ) -> StorageSessionABC: + storage_session_id = ( + f"{self._session_id}_storage_{uuid.uuid1()}" + if not storage_session_id + else storage_session_id + ) + + if storage_session_id in self._storage_session: + return self._storage_session[storage_session_id] + else: + if storage_engine is None: + storage_engine = self._storage_engine + + for session in self._storage_session.values(): + if storage_engine == session.engine: + return session + + if record: + self.save_record( + engine_type=EngineType.STORAGE, + engine_name=storage_engine, + engine_session_id=storage_session_id, + ) + if storage_engine == StorageEngine.EGGROLL: + from fate_flow.engine.storage.eggroll import StorageSession + kwargs["host"] = COMPUTING_CONF.get(StorageEngine.EGGROLL).get("host") + kwargs["port"] = COMPUTING_CONF.get(StorageEngine.EGGROLL).get("port") + + elif storage_engine == StorageEngine.STANDALONE: + from fate_flow.engine.storage.standalone import StorageSession + + elif storage_engine == StorageEngine.FILE: + from fate_flow.engine.storage.file import StorageSession + + elif storage_engine == StorageEngine.HDFS: + from fate_flow.engine.storage.hdfs import StorageSession + + else: + raise NotImplementedError( + f"can not be initialized with storage engine: {storage_engine}" + ) + storage_session = StorageSession( + session_id=storage_session_id, **kwargs + ) + + self._storage_session[storage_session_id] = storage_session + + return storage_session + + def get_table( + self, name, namespace, ignore_disable=False + ) -> typing.Union[StorageTableABC, None]: + meta = Session.get_table_meta(name=name, namespace=namespace) + if meta is None: + return None + if meta.get_disable() and not ignore_disable: + raise Exception(f"table {namespace} {name} disable: {meta.get_disable()}") + engine = meta.get_engine() + storage_session = self._get_or_create_storage(storage_engine=engine) + table = storage_session.get_table(name=name, namespace=namespace) + return table + + @classmethod + def get_table_meta(cls, name, namespace) -> typing.Union[StorageTableMetaABC, None]: + meta = StorageSessionBase.get_table_meta(name=name, namespace=namespace) + return meta + + def storage(self, **kwargs): + return self._get_or_create_storage(**kwargs) + + @DB.connection_context() + def save_record(self, engine_type, engine_name, engine_session_id): + self._logger.info( + f"try to save session record for manager {self._session_id}, {engine_type} {engine_name} {engine_session_id}" + ) + session_record = SessionRecord() + session_record.f_manager_session_id = self._session_id + session_record.f_engine_type = engine_type + session_record.f_engine_name = engine_name + session_record.f_engine_session_id = engine_session_id + # TODO: engine address + session_record.f_engine_address = {} + session_record.f_create_time = base_utils.current_timestamp() + msg = f"save storage session record for manager {self._session_id}, {engine_type} {engine_name} {engine_session_id}" + try: + effect_count = session_record.save(force_insert=True) + if effect_count != 1: + raise RuntimeError(f"{msg} failed") + except peewee.IntegrityError as e: + LOGGER.warning(e) + except Exception as e: + raise RuntimeError(f"{msg} exception", e) + self._logger.info( + f"save session record for manager {self._session_id}, {engine_type} {engine_name} {engine_session_id} successfully" + ) + + @DB.connection_context() + def delete_session_record(self, engine_session_id): + rows = ( + SessionRecord.delete() + .where(SessionRecord.f_engine_session_id == engine_session_id) + .execute() + ) + if rows > 0: + self._logger.info(f"delete session {engine_session_id} record successfully") + else: + self._logger.warning(f"delete session {engine_session_id} record failed") + + @classmethod + @DB.connection_context() + def query_sessions(cls, reverse=None, order_by=None, **kwargs): + return SessionRecord.query(reverse=reverse, order_by=order_by, **kwargs) + + @DB.connection_context() + def get_session_from_record(self, **kwargs): + self._logger.info(f"query by manager session id {self._session_id}") + session_records = self.query_sessions( + manager_session_id=self._session_id, **kwargs + ) + self._logger.info( + [session_record.f_engine_session_id for session_record in session_records] + ) + for session_record in session_records: + try: + engine_session_id = session_record.f_engine_session_id + if session_record.f_engine_type == EngineType.STORAGE: + self._get_or_create_storage( + storage_session_id=engine_session_id, + storage_engine=session_record.f_engine_name, + record=False, + ) + except Exception as e: + self._logger.error(e) + self.delete_session_record( + engine_session_id=session_record.f_engine_session_id + ) + + def destroy_all_sessions(self, **kwargs): + self._logger.info( + f"start destroy manager session {self._session_id} all sessions" + ) + self.get_session_from_record(**kwargs) + self.destroy_storage_session() + self._logger.info( + f"finish destroy manager session {self._session_id} all sessions" + ) + + def destroy_storage_session(self): + for session_id, session in self._storage_session.items(): + try: + self._logger.info(f"try to destroy storage session {session_id}") + session.destroy() + self._logger.info(f"destroy storage session {session_id} successfully") + except Exception as e: + self._logger.exception( + f"destroy storage session {session_id} failed", e + ) + self.delete_session_record(engine_session_id=session_id) + + +def get_session() -> Session: + return Session.get_global() diff --git a/python/fate_flow/engine/storage/_table.py b/python/fate_flow/engine/storage/_table.py new file mode 100644 index 000000000..642fb1e49 --- /dev/null +++ b/python/fate_flow/engine/storage/_table.py @@ -0,0 +1,483 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import operator +from typing import Iterable, Tuple + +import peewee + +from fate_flow.db.base_models import DB +from fate_flow.db.storage_models import StorageTableMetaModel +from fate_flow.engine.relation_ship import Relationship +from fate_flow.engine.storage._abc import StorageTableMetaABC, StorageTableABC +from fate_flow.entity.types import AddressABC +from fate_flow.utils.base_utils import current_timestamp +from fate_flow.utils.log import getLogger +from ._partitioner import get_partitioner_by_type +from .serdes import get_serdes_by_type + +LOGGER = getLogger("storage") + + +def _wrapped_iterable_with_serdes( + kv_list: Iterable[Tuple[bytes, bytes]], key_serdes, value_serdes +): + for k, v in kv_list: + yield key_serdes.serialize(k), value_serdes.serialize(v) + + +class StorageTableBase(StorageTableABC): + def __init__( + self, + name, + namespace, + address, + partitions, + options, + engine, + key_serdes_type, + value_serdes_type, + partitioner_type, + ): + self._name = name + self._namespace = namespace + self._address = address + self._partitions = partitions + self._options = options if options else {} + self._engine = engine + self._key_serdes_type = key_serdes_type + self._value_serdes_type = value_serdes_type + self._partitioner_type = partitioner_type + self._key_serdes = None + self._value_serdes = None + self._partitioner = None + + self._meta = None + self._read_access_time = None + self._write_access_time = None + + @property + def key_serdes(self): + if self._key_serdes is None: + self._key_serdes = get_serdes_by_type(self._key_serdes_type) + return self._key_serdes + + @property + def value_serdes(self): + if self._value_serdes is None: + self._value_serdes = get_serdes_by_type(self._value_serdes_type) + return self._value_serdes + + @property + def partitioner(self): + if self._partitioner is None: + self._partitioner = get_partitioner_by_type(self._partitioner_type) + return self._partitioner + + @property + def name(self): + return self._name + + @property + def namespace(self): + return self._namespace + + @property + def address(self): + return self._address + + @property + def partitions(self): + return self._partitions + + @property + def data_type(self): + return self.meta.data_type + + @property + def options(self): + return self._options + + @property + def engine(self): + return self._engine + + @property + def meta(self): + return self._meta + + @meta.setter + def meta(self, meta): + self._meta = meta + + @property + def read_access_time(self): + return self._read_access_time + + @property + def write_access_time(self): + return self._write_access_time + + def update_meta( + self, + data_meta=None, + count=None, + part_of_data=None, + description=None, + partitions=None, + **kwargs + ): + self._meta.update_metas( + data_meta=data_meta, + count=count, + part_of_data=part_of_data, + description=description, + partitions=partitions, + **kwargs + ) + + def create_meta(self, **kwargs): + self.destroy_if_exists() + table_meta = StorageTableMeta( + name=self._name, namespace=self._namespace, new=True + ) + table_meta.set_metas(**kwargs) + table_meta.address = self._address + table_meta.partitions = self._partitions + table_meta.engine = self._engine + table_meta.options = self._options + table_meta.create() + self._meta = table_meta + + return table_meta + + def destroy_if_exists(self): + table_meta = StorageTableMeta(name=self._name, namespace=self._namespace) + if table_meta: + table_meta.destroy_metas() + return True + return False + + def check_address(self): + return True + + def put_all(self, kv_list: Iterable, **kwargs): + # self._update_write_access_time() + self._put_all( + _wrapped_iterable_with_serdes(kv_list, self.key_serdes, self.value_serdes), + self.partitioner, + **kwargs + ) + + def collect(self, **kwargs) -> list: + # self._update_read_access_time() + for k, v in self._collect(**kwargs): + yield self.key_serdes.deserialize(k), self.value_serdes.deserialize(v) + + def count(self): + # self._update_read_access_time() + count = self._count() + self.meta.update_metas(count=count) + return count + + def read(self): + # self._update_read_access_time() + return self._read() + + def destroy(self): + self.meta.destroy_metas() + self._destroy() + + # to be implemented + def _put_all(self, kv_list: Iterable[Tuple[bytes, bytes]], partitioner, **kwargs): + raise NotImplementedError() + + def _collect(self, **kwargs) -> list: + raise NotImplementedError() + + def _count(self): + raise NotImplementedError() + + def _read(self): + raise NotImplementedError() + + def _destroy(self): + raise NotImplementedError() + + def _save_as( + self, address, name, namespace, partitions=None, schema=None, **kwargs + ): + raise NotImplementedError() + + +class StorageTableMeta(StorageTableMetaABC): + def __init__(self, name, namespace, new=False, create_address=True): + self.name = name + self.namespace = namespace + self.address = None + self.engine = None + self.store_type = None + self.options = None + self.partitions = None + self.have_head = None + self.extend_sid = False + self.auto_increasing_sid = None + self.data_meta = None + self.data_type = None + self.count = None + self.part_of_data = None + self.description = None + self.source = None + self.disable = None + self.create_time = None + self.update_time = None + self.read_access_time = None + self.write_access_time = None + if self.options is None: + self.options = {} + if self.data_meta is None: + self.data_meta = {} + if self.part_of_data is None: + self.part_of_data = [] + if not new: + self.build(create_address) + + def build(self, create_address): + for k, v in self.table_meta.__dict__["__data__"].items(): + setattr(self, k.lstrip("f_"), v) + if create_address: + self.address = self.create_address( + storage_engine=self.engine, address_dict=self.address + ) + + def __new__(cls, *args, **kwargs): + if not kwargs.get("new", False): + name, namespace = kwargs.get("name"), kwargs.get("namespace") + if not name or not namespace: + return None + tables_meta = cls.query_table_meta( + filter_fields=dict(name=name, namespace=namespace) + ) + if not tables_meta: + return None + self = super().__new__(cls) + setattr(self, "table_meta", tables_meta[0]) + return self + else: + return super().__new__(cls) + + def exists(self): + if hasattr(self, "table_meta"): + return True + else: + return False + + @DB.connection_context() + def create(self): + table_meta = StorageTableMetaModel() + table_meta.f_create_time = current_timestamp() + table_meta.f_data_meta = {} + table_meta.f_part_of_data = [] + table_meta.f_source = {} + for k, v in self.to_dict().items(): + attr_name = "f_%s" % k + if hasattr(StorageTableMetaModel, attr_name): + setattr( + table_meta, + attr_name, + v if not issubclass(type(v), AddressABC) else v.__dict__, + ) + try: + rows = table_meta.save(force_insert=True) + if rows != 1: + raise Exception("create table meta failed") + except peewee.IntegrityError as e: + # if e.args[0] == 1062: + # # warning + # pass + # elif isinstance(e.args[0], str) and "UNIQUE constraint failed" in e.args[0]: + # pass + # else: + # raise e + pass + except Exception as e: + raise e + + def set_metas(self, **kwargs): + for k, v in kwargs.items(): + if hasattr(self, k): + setattr(self, k, v) + + @classmethod + @DB.connection_context() + def query_table_meta(cls, filter_fields, query_fields=None): + filters = [] + querys = [] + for f_n, f_v in filter_fields.items(): + attr_name = "f_%s" % f_n + if hasattr(StorageTableMetaModel, attr_name): + filters.append( + operator.attrgetter("f_%s" % f_n)(StorageTableMetaModel) == f_v + ) + if query_fields: + for f_n in query_fields: + attr_name = "f_%s" % f_n + if hasattr(StorageTableMetaModel, attr_name): + querys.append( + operator.attrgetter("f_%s" % f_n)(StorageTableMetaModel) + ) + if filters: + if querys: + tables_meta = StorageTableMetaModel.select(querys).where(*filters) + else: + tables_meta = StorageTableMetaModel.select().where(*filters) + return [table_meta for table_meta in tables_meta] + else: + # not allow query all table + return [] + + @DB.connection_context() + def update_metas( + self, + data_meta=None, + count=None, + part_of_data=None, + description=None, + partitions=None, + in_serialized=None, + **kwargs + ): + meta_info = {} + for k, v in locals().items(): + if k not in ["self", "kwargs", "meta_info"] and v is not None: + meta_info[k] = v + meta_info.update(kwargs) + meta_info["name"] = meta_info.get("name", self.name) + meta_info["namespace"] = meta_info.get("namespace", self.namespace) + update_filters = [] + primary_keys = StorageTableMetaModel._meta.primary_key.field_names + for p_k in primary_keys: + update_filters.append( + operator.attrgetter(p_k)(StorageTableMetaModel) + == meta_info[p_k.lstrip("f_")] + ) + table_meta = StorageTableMetaModel() + update_fields = {} + for k, v in meta_info.items(): + attr_name = "f_%s" % k + if ( + hasattr(StorageTableMetaModel, attr_name) + and attr_name not in primary_keys + ): + if k == "part_of_data": + if len(v) < 100: + tmp = v + else: + tmp = v[:100] + update_fields[ + operator.attrgetter(attr_name)(StorageTableMetaModel) + ] = tmp + else: + update_fields[ + operator.attrgetter(attr_name)(StorageTableMetaModel) + ] = v + if update_filters: + operate = table_meta.update(update_fields).where(*update_filters) + else: + operate = table_meta.update(update_fields) + if count: + self.count = count + _return = operate.execute() + _meta = StorageTableMeta(name=self.name, namespace=self.namespace) + return _return > 0, _meta + + @DB.connection_context() + def destroy_metas(self): + StorageTableMetaModel.delete().where( + StorageTableMetaModel.f_name == self.name, + StorageTableMetaModel.f_namespace == self.namespace, + ).execute() + + @classmethod + def create_address(cls, storage_engine, address_dict): + address_class = Relationship.EngineToAddress.get(storage_engine) + kwargs = {} + for k in address_class.__init__.__code__.co_varnames: + if k == "self": + continue + if address_dict.get(k, None): + kwargs[k] = address_dict[k] + return address_class(**kwargs) + + def get_name(self): + return self.name + + def get_namespace(self): + return self.namespace + + def get_address(self): + return self.address + + def get_engine(self): + return self.engine + + def get_store_type(self): + return self.store_type + + def get_options(self): + return self.options + + def get_partitions(self): + return self.partitions + + def get_id_delimiter(self): + return self.data_meta.get("delimiter", ",") + + def get_extend_sid(self): + return self.extend_sid + + def get_auto_increasing_sid(self): + return self.auto_increasing_sid + + def get_have_head(self): + return self.have_head + + def get_source(self): + return self.source + + def get_disable(self): + return self.disable + + def get_data_meta(self): + return self.data_meta + + def get_count(self): + return self.count + + def get_part_of_data(self): + return self.part_of_data + + def get_description(self): + return self.description + + def to_dict(self) -> dict: + d = {} + for k, v in self.__dict__.items(): + if v is None or k == "table_meta": + continue + d[k] = v + return d diff --git a/python/fate_flow/engine/storage/_types.py b/python/fate_flow/engine/storage/_types.py new file mode 100644 index 000000000..b09c644c0 --- /dev/null +++ b/python/fate_flow/engine/storage/_types.py @@ -0,0 +1,117 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +DEFAULT_ID_DELIMITER = "," + + +class DataType: + TABLE = "table" + DATAFRAME = "dataframe" + FILE = "file" + DATA_DIRECTORY = "data_directory" + DATA_UNRESOLVED = "data_unresolved" + + +class StorageOrigin(object): + TABLE_BIND = "table_bind" + READER = "reader" + UPLOAD = "upload" + OUTPUT = "output" + + +class StorageEngine(object): + STANDALONE = 'standalone' + EGGROLL = 'eggroll' + HDFS = 'hdfs' + MYSQL = 'mysql' + SIMPLE = 'simple' + PATH = 'path' + FILE = 'file' + HIVE = 'hive' + API = 'api' + HTTP = 'http' + HTTPS = 'https' + + +class StandaloneStoreType(object): + ROLLPAIR_IN_MEMORY = 'IN_MEMORY' + ROLLPAIR_LMDB = 'LMDB' + DEFAULT = ROLLPAIR_LMDB + + +class EggRollStoreType(object): + ROLLPAIR_IN_MEMORY = 'IN_MEMORY' + ROLLPAIR_LMDB = 'LMDB' + ROLLPAIR_LEVELDB = 'LEVEL_DB' + ROLLFRAME_FILE = 'ROLL_FRAME_FILE' + ROLLPAIR_ROLLSITE = 'ROLL_SITE' + ROLLPAIR_FILE = 'ROLL_PAIR_FILE' + ROLLPAIR_MMAP = 'ROLL_PAIR_MMAP' + ROLLPAIR_CACHE = 'ROLL_PAIR_CACHE' + ROLLPAIR_QUEUE = 'ROLL_PAIR_QUEUE' + DEFAULT = ROLLPAIR_LMDB + + +class HDFSStoreType(object): + RAM_DISK = 'RAM_DISK' + SSD = 'SSD' + DISK = 'DISK' + ARCHIVE = 'ARCHIVE' + DEFAULT = None + + +class PathStoreType(object): + PICTURE = 'PICTURE' + + +class FileStoreType(object): + CSV = 'CSV' + + +class ApiStoreType(object): + EXTERNAL = 'EXTERNAL' + + +class MySQLStoreType(object): + InnoDB = "InnoDB" + MyISAM = "MyISAM" + ISAM = "ISAM" + HEAP = "HEAP" + DEFAULT = None + + +class HiveStoreType(object): + DEFAULT = "HDFS" + + +class LinkisHiveStoreType(object): + DEFAULT = "HDFS" + + +class LocalFSStoreType(object): + RAM_DISK = 'RAM_DISK' + SSD = 'SSD' + DISK = 'DISK' + ARCHIVE = 'ARCHIVE' + DEFAULT = None + + +class StorageTableMetaType(object): + ENGINE = "engine" + TYPE = "type" + SCHEMA = "schema" + PART_OF_DATA = "part_of_data" + COUNT = "count" + PARTITIONS = "partitions" diff --git a/python/fate_flow/engine/storage/eggroll/__init__.py b/python/fate_flow/engine/storage/eggroll/__init__.py new file mode 100644 index 000000000..98046cca3 --- /dev/null +++ b/python/fate_flow/engine/storage/eggroll/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.engine.storage.eggroll._table import StorageTable +from fate_flow.engine.storage.eggroll._session import StorageSession + +__all__ = ["StorageTable", "StorageSession"] diff --git a/python/fate_flow/engine/storage/eggroll/_session.py b/python/fate_flow/engine/storage/eggroll/_session.py new file mode 100644 index 000000000..d048eb253 --- /dev/null +++ b/python/fate_flow/engine/storage/eggroll/_session.py @@ -0,0 +1,127 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from eggroll.session import session_init +from eggroll.computing import RollPairContext +from fate_flow.engine.storage import EggRollStoreType, StorageEngine, StorageSessionBase +from fate_flow.engine.storage.eggroll._table import StorageTable +from fate_flow.entity.types import EggRollAddress + + +class StorageSession(StorageSessionBase): + def __init__(self, session_id, host: str = None, port: int = None, options=None, config_options=None, config_properties_file=None): + super(StorageSession, self).__init__( + session_id=session_id, engine=StorageEngine.EGGROLL + ) + self._options = options if options else {} + self._rp_session = session_init( + session_id=self._session_id, + host=host, + port=port, + options=self._options, + config_options=config_options, + config_properties_file=config_properties_file + ) + self._rpc = RollPairContext(session=self._rp_session) + self._session_id = self._rp_session.get_session_id() + + def load( + self, + name, + namespace, + address: EggRollAddress, + store_type, + partitions, + options=None, + **kwargs, + ): + if isinstance(address, EggRollAddress): + _table = self._rpc.load_rp( + namespace=address.namespace, + name=address.name, + store_type=store_type, + ) + + return StorageTable( + context=self._rpc, + table=_table, + name=name, + namespace=namespace, + address=address, + partitions=partitions, + store_type=store_type, + key_serdes_type=_table.get_store().key_serdes_type, + value_serdes_type=_table.get_store().value_serdes_type, + partitioner_type=_table.get_store().partitioner_type, + options=options, + ) + raise NotImplementedError( + f"address type {type(address)} not supported with standalone storage" + ) + + def table( + self, + name, + namespace, + address, + partitions, + key_serdes_type, + value_serdes_type, + partitioner_type, + store_type: str = EggRollStoreType.ROLLPAIR_LMDB, + options=None, + **kwargs, + ): + if isinstance(address, EggRollAddress): + if options is None: + options = {} + _table = self._rpc.create_rp( + id=-1, + name=address.name, + namespace=address.namespace, + total_partitions=partitions, + store_type=store_type, + key_serdes_type=key_serdes_type, + value_serdes_type=value_serdes_type, + partitioner_type=partitioner_type, + options=options, + ) + + return StorageTable( + context=self._rpc, + table=_table, + key_serdes_type=key_serdes_type, + value_serdes_type=value_serdes_type, + partitioner_type=partitioner_type, + name=name, + namespace=namespace, + address=address, + partitions=partitions, + store_type=store_type, + options=options, + ) + raise NotImplementedError( + f"address type {type(address)} not supported with eggroll storage" + ) + + def cleanup(self, name, namespace): + self._rpc.cleanup(name=name, namespace=namespace) + + def stop(self): + return self._rp_session.stop() + + def kill(self): + return self._rp_session.kill() diff --git a/python/fate_flow/engine/storage/eggroll/_table.py b/python/fate_flow/engine/storage/eggroll/_table.py new file mode 100644 index 000000000..48d80d58d --- /dev/null +++ b/python/fate_flow/engine/storage/eggroll/_table.py @@ -0,0 +1,77 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Iterable +from fate_flow.engine.storage import StorageTableBase, EggRollStoreType, StorageEngine +from eggroll.computing import RollPairContext, RollPair + + +class StorageTable(StorageTableBase): + def __init__( + self, + context: RollPairContext, + table: RollPair, + key_serdes_type, + value_serdes_type, + partitioner_type, + name, + namespace, + address, + partitions: int = 1, + store_type: str = EggRollStoreType.ROLLPAIR_LMDB, + options=None, + ): + self._context = context + self._store_type = store_type + self._table = table + super(StorageTable, self).__init__( + name=name, + namespace=namespace, + address=address, + partitions=partitions, + options=options, + engine=StorageEngine.EGGROLL, + key_serdes_type=key_serdes_type, + value_serdes_type=value_serdes_type, + partitioner_type=partitioner_type, + ) + self._options["store_type"] = self._store_type + self._options["total_partitions"] = partitions + self._options["create_if_missing"] = True + + # + # def _save_as(self, address, name, namespace, partitions=None, **kwargs): + # self._table.save_as(name=address.name, namespace=address.namespace) + # table = StorageTable( + # context=self._context, + # address=address, + # partitions=partitions, + # name=name, + # namespace=namespace, + # ) + # return table + + def _put_all(self, kv_list: Iterable, partitioner, **kwargs): + return self._table.put_all(kv_list, partitioner) + + def _collect(self, **kwargs) -> list: + return self._table.get_all(**kwargs) + + def _destroy(self): + self._table.destroy() + + def _count(self, **kwargs): + return self._table.count() diff --git a/python/fate_flow/scheduling_apps/client/__init__.py b/python/fate_flow/engine/storage/file/__init__.py similarity index 77% rename from python/fate_flow/scheduling_apps/client/__init__.py rename to python/fate_flow/engine/storage/file/__init__.py index 3b2f0af28..786c7bcca 100644 --- a/python/fate_flow/scheduling_apps/client/__init__.py +++ b/python/fate_flow/engine/storage/file/__init__.py @@ -13,5 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from fate_flow.scheduling_apps.client.control_client import ControllerClient -from fate_flow.scheduling_apps.client.tracker_client import TrackerClient +from fate_flow.engine.storage.file._session import StorageSession +from fate_flow.engine.storage.file._table import StorageTable + +__all__ = ["StorageTable", "StorageSession"] diff --git a/python/fate_flow/engine/storage/file/_session.py b/python/fate_flow/engine/storage/file/_session.py new file mode 100644 index 000000000..6bb8d5ee6 --- /dev/null +++ b/python/fate_flow/engine/storage/file/_session.py @@ -0,0 +1,53 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.engine.storage import StorageSessionBase, StorageEngine +from fate_flow.engine.storage.file._table import StorageTable +from fate_flow.entity.types import AddressABC, FileAddress + + +class StorageSession(StorageSessionBase): + def __init__(self, session_id, options=None): + super(StorageSession, self).__init__(session_id=session_id, engine=StorageEngine.FILE) + + def table(self, address: AddressABC, name, namespace, partitions, storage_type=None, options=None, **kwargs): + if isinstance(address, FileAddress): + return StorageTable(address=address, name=name, namespace=namespace, + partitions=partitions) + raise NotImplementedError(f"address type {type(address)} not supported with hdfs storage") + + def load( + self, + name, + namespace, + address, + store_type, + partitions, + options=None, + **kwargs, + ): + if isinstance(address, FileAddress): + return StorageTable(address=address, name=name, namespace=namespace, + partitions=partitions, options=options) + raise NotImplementedError(f"address type {type(address)} not supported with hdfs storage") + + def cleanup(self, name, namespace): + pass + + def stop(self): + pass + + def kill(self): + pass diff --git a/python/fate_flow/engine/storage/file/_table.py b/python/fate_flow/engine/storage/file/_table.py new file mode 100644 index 000000000..1ea703c43 --- /dev/null +++ b/python/fate_flow/engine/storage/file/_table.py @@ -0,0 +1,149 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import io +import os +import struct +from typing import Iterable, Tuple + +from pyarrow import fs + +from fate_flow.engine.storage import StorageTableBase, StorageEngine +from fate_flow.utils.log import getLogger + +LOGGER = getLogger() + +class FileCoder: + @staticmethod + def encode(key: bytes, value: bytes): + size = struct.pack(">Q", len(key)) + return (size + key + value).hex() + + @staticmethod + def decode(data: str) -> Tuple[bytes, bytes]: + data = bytes.fromhex(data) + size = struct.unpack(">Q", data[:8])[0] + key = data[8 : 8 + size] + value = data[8 + size :] + return key, value + + +class StorageTable(StorageTableBase): + def __init__( + self, + address=None, + name: str = None, + namespace: str = None, + partitions: int = 1, + options=None, + ): + super(StorageTable, self).__init__( + name=name, + namespace=namespace, + address=address, + partitions=partitions, + options=options, + engine=StorageEngine.FILE, + key_serdes_type=0, + value_serdes_type=0, + partitioner_type=0 + ) + self._local_fs_client = fs.LocalFileSystem() + + @property + def path(self): + return self._address.path + + def _put_all( + self, kv_list: Iterable, append=True, assume_file_exist=False, **kwargs + ): + LOGGER.info(f"put in file: {self.path}") + + self._local_fs_client.create_dir(os.path.dirname(self.path)) + + if append and (assume_file_exist or self._exist()): + stream = self._local_fs_client.open_append_stream( + path=self.path, compression=None + ) + else: + stream = self._local_fs_client.open_output_stream( + path=self.path, compression=None + ) + + counter = self._meta.get_count() if self._meta.get_count() else 0 + with io.TextIOWrapper(stream) as writer: + for k, v in kv_list: + writer.write(FileCoder.encode(k, v)) + writer.write("\n") + counter = counter + 1 + self._meta.update_metas(count=counter) + + def _collect(self, **kwargs) -> list: + for line in self._as_generator(): + yield FileCoder.decode(line.rstrip()) + + def _read(self) -> list: + for line in self._as_generator(): + yield line + + def _destroy(self): + # use try/catch to avoid stop while deleting an non-exist file + try: + self._local_fs_client.delete_file(self.path) + except Exception as e: + LOGGER.debug(e) + + def _count(self): + count = 0 + for _ in self._as_generator(): + count += 1 + return count + + def close(self): + pass + + def _exist(self): + info = self._local_fs_client.get_file_info([self.path])[0] + return info.type != fs.FileType.NotFound + + def _as_generator(self): + info = self._local_fs_client.get_file_info([self.path])[0] + if info.type == fs.FileType.NotFound: + raise FileNotFoundError(f"file {self.path} not found") + + elif info.type == fs.FileType.File: + with io.TextIOWrapper( + buffer=self._local_fs_client.open_input_stream(self.path), encoding="utf-8" + ) as reader: + for line in reader: + yield line + else: + selector = fs.FileSelector(self.path) + file_infos = self._local_fs_client.get_file_info(selector) + for file_info in file_infos: + if file_info.base_name.startswith(".") or file_info.base_name.startswith("_"): + continue + assert ( + file_info.is_file + ), f"{self.path} is directory contains a subdirectory: {file_info.path}" + with io.TextIOWrapper( + buffer=self._local_fs_client.open_input_stream( + f"{self._address.file_path:}/{file_info.path}" + ), + encoding="utf-8", + ) as reader: + for line in reader: + yield line diff --git a/python/fate_flow/engine/storage/hdfs/__init__.py b/python/fate_flow/engine/storage/hdfs/__init__.py new file mode 100644 index 000000000..31c1dbd35 --- /dev/null +++ b/python/fate_flow/engine/storage/hdfs/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.engine.storage.hdfs._table import StorageTable +from fate_flow.engine.storage.hdfs._session import StorageSession + +__all__ = ["StorageTable", "StorageSession"] diff --git a/python/fate_flow/engine/storage/hdfs/_session.py b/python/fate_flow/engine/storage/hdfs/_session.py new file mode 100644 index 000000000..e9f170360 --- /dev/null +++ b/python/fate_flow/engine/storage/hdfs/_session.py @@ -0,0 +1,63 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.engine.storage import StorageSessionBase, StorageEngine +from fate_flow.engine.storage.hdfs._table import StorageTable +from fate_flow.entity.types import AddressABC, HDFSAddress + + +class StorageSession(StorageSessionBase): + def __init__(self, session_id, options=None): + super(StorageSession, self).__init__(session_id=session_id, engine=StorageEngine.HDFS) + + def table(self, address: AddressABC, name, namespace, partitions, store_type=None, options=None, **kwargs): + if isinstance(address, HDFSAddress): + return StorageTable( + address=address, + name=name, + namespace=namespace, + partitions=partitions, + options=options + ) + raise NotImplementedError(f"address type {type(address)} not supported with hdfs storage") + + def load( + self, + name, + namespace, + address, + store_type, + partitions, + options=None, + **kwargs, + ): + if isinstance(address, HDFSAddress): + return StorageTable( + address=address, + name=name, + namespace=namespace, + partitions=partitions, + options=options + ) + raise NotImplementedError(f"address type {type(address)} not supported with hdfs storage") + + def cleanup(self, name, namespace): + pass + + def stop(self): + pass + + def kill(self): + pass diff --git a/python/fate_flow/engine/storage/hdfs/_table.py b/python/fate_flow/engine/storage/hdfs/_table.py new file mode 100644 index 000000000..5ad70a2f4 --- /dev/null +++ b/python/fate_flow/engine/storage/hdfs/_table.py @@ -0,0 +1,169 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import io +from typing import Iterable, Tuple + +from pyarrow import fs + +from fate_flow.engine.storage import StorageTableBase +from fate_flow.engine.storage._types import StorageEngine +from fate_flow.utils.log import getLogger +import struct + + +LOGGER = getLogger() + + +class HDFSCoder: + @staticmethod + def encode(key: bytes, value: bytes): + size = struct.pack(">Q", len(key)) + return (size + key + value).hex() + + @staticmethod + def decode(data: str) -> Tuple[bytes, bytes]: + data = bytes.fromhex(data) + size = struct.unpack(">Q", data[:8])[0] + key = data[8 : 8 + size] + value = data[8 + size :] + return key, value + + +class StorageTable(StorageTableBase): + def __init__( + self, + address=None, + name: str = None, + namespace: str = None, + partitions: int = 1, + options=None, + ): + super(StorageTable, self).__init__( + name=name, + namespace=namespace, + address=address, + partitions=partitions, + options=options, + engine=StorageEngine.HDFS, + key_serdes_type=0, + value_serdes_type=0, + partitioner_type=0, + ) + try: + # noinspection PyUnresolvedReferences + from pyarrow import HadoopFileSystem + HadoopFileSystem(self.path) + except Exception as e: + LOGGER.warning(f"load libhdfs failed: {e}") + + # pyarrow.fs.HadoopFileSystem.from_uri(uri, **kwargs) supports the following formats: + # * ``HadoopFileSystem.from_uri('hdfs://localhost:8020/?user=test&replication=1')`` + # * ``HadoopFileSystem('localhost', port=8020, user='test', replication=1)`` + # your IDE may complain about the following line, but it works. + # noinspection PyArgumentList + self._hdfs_client = fs.HadoopFileSystem.from_uri(self.path) + + def check_address(self): + return self._exist() + + def _put_all( + self, kv_list: Iterable, append=True, assume_file_exist=False, **kwargs + ): + + client = self._hdfs_client + path = self.file_path + LOGGER.info(f"put in hdfs file: {path}") + if append and (assume_file_exist or self._exist(path)): + stream = client.open_append_stream( + path=path, compression=None + ) + else: + stream = client.open_output_stream( + path=path, compression=None + ) + + counter = self._meta.get_count() if self._meta.get_count() else 0 + with io.TextIOWrapper(stream) as writer: + for k, v in kv_list: + writer.write(HDFSCoder.encode(k, v)) + writer.write("\n") + counter = counter + 1 + self._meta.update_metas(count=counter) + + def _collect(self, **kwargs) -> list: + for line in self._as_generator(): + yield HDFSCoder.decode(line.rstrip()) + + def _read(self) -> list: + for line in self._as_generator(): + yield line + + def _destroy(self): + self._hdfs_client.delete_file(self.file_path) + + def _count(self): + count = 0 + if self._meta.get_count(): + return self._meta.get_count() + for _ in self._as_generator(): + count += 1 + return count + + def close(self): + pass + + @property + def path(self) -> str: + return f"{self._address.name_node}/{self._address.path}" + + @property + def file_path(self) -> str: + return f"{self._address.path}" + + def _exist(self, path=None): + if not path: + path = self.file_path + info = self._hdfs_client.get_file_info([path])[0] + return info.type != fs.FileType.NotFound + + def _as_generator(self): + file = self.file_path + LOGGER.info(f"as generator: {file}") + info = self._hdfs_client.get_file_info([file])[0] + if info.type == fs.FileType.NotFound: + raise FileNotFoundError(f"file {file} not found") + + elif info.type == fs.FileType.File: + with io.TextIOWrapper( + buffer=self._hdfs_client.open_input_stream(self.path), encoding="utf-8" + ) as reader: + for line in reader: + yield line + else: + selector = fs.FileSelector(file) + file_infos = self._hdfs_client.get_file_info(selector) + for file_info in file_infos: + if file_info.base_name == "_SUCCESS": + continue + assert ( + file_info.is_file + ), f"{self.path} is directory contains a subdirectory: {file_info.path}" + with io.TextIOWrapper( + buffer=self._hdfs_client.open_input_stream(file_info.path), + encoding="utf-8", + ) as reader: + for line in reader: + yield line diff --git a/python/fate_flow/engine/storage/serdes/__init__.py b/python/fate_flow/engine/storage/serdes/__init__.py new file mode 100644 index 000000000..f2b558642 --- /dev/null +++ b/python/fate_flow/engine/storage/serdes/__init__.py @@ -0,0 +1,28 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +def get_serdes_by_type(serdes_type: int): + if serdes_type == 0: + from ._unrestricted_serdes import get_unrestricted_serdes + + return get_unrestricted_serdes() + elif serdes_type == 1: + from ._integer_serdes import get_integer_serdes + + return get_integer_serdes() + else: + raise ValueError(f"serdes type `{serdes_type}` not supported") diff --git a/python/fate_flow/engine/storage/serdes/_integer_serdes.py b/python/fate_flow/engine/storage/serdes/_integer_serdes.py new file mode 100644 index 000000000..dd1674128 --- /dev/null +++ b/python/fate_flow/engine/storage/serdes/_integer_serdes.py @@ -0,0 +1,30 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +def get_integer_serdes(): + return IntegerSerdes() + + +class IntegerSerdes: + def __init__(self): + ... + + def serialize(self, obj) -> bytes: + return obj.to_bytes(8, "big") + + def deserialize(self, bytes) -> object: + return int.from_bytes(bytes, "big") diff --git a/python/fate_flow/engine/storage/serdes/_serdes_base.py b/python/fate_flow/engine/storage/serdes/_serdes_base.py new file mode 100644 index 000000000..06e5d0987 --- /dev/null +++ b/python/fate_flow/engine/storage/serdes/_serdes_base.py @@ -0,0 +1,21 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from pickle import loads as p_loads +from pickle import dumps as p_dumps +from pickle import Pickler, Unpickler + +__all__ = ["p_dumps", "p_loads", "Pickler", "Unpickler"] \ No newline at end of file diff --git a/python/fate_flow/engine/storage/serdes/_unrestricted_serdes.py b/python/fate_flow/engine/storage/serdes/_unrestricted_serdes.py new file mode 100644 index 000000000..df84f9250 --- /dev/null +++ b/python/fate_flow/engine/storage/serdes/_unrestricted_serdes.py @@ -0,0 +1,31 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from ._serdes_base import p_dumps, p_loads + + +def get_unrestricted_serdes(): + return UnrestrictedSerdes + + +class UnrestrictedSerdes: + @staticmethod + def serialize(obj) -> bytes: + return p_dumps(obj) + + @staticmethod + def deserialize(bytes) -> object: + return p_loads(bytes) diff --git a/python/fate_flow/engine/storage/standalone/__init__.py b/python/fate_flow/engine/storage/standalone/__init__.py new file mode 100644 index 000000000..1e1905218 --- /dev/null +++ b/python/fate_flow/engine/storage/standalone/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.engine.storage.standalone._table import StorageTable +from fate_flow.engine.storage.standalone._session import StorageSession + +__all__ = ["StorageTable", "StorageSession"] diff --git a/python/fate_flow/engine/storage/standalone/_session.py b/python/fate_flow/engine/storage/standalone/_session.py new file mode 100644 index 000000000..4046632f6 --- /dev/null +++ b/python/fate_flow/engine/storage/standalone/_session.py @@ -0,0 +1,121 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +from fate_flow.engine.storage import ( + StorageSessionBase, + StorageEngine, + StandaloneStoreType, +) +from fate_flow.engine.storage.standalone._table import StorageTable +from fate_flow.engine.storage.standalone._standalone import Session +from fate_flow.entity.types import AddressABC, StandaloneAddress +from fate_flow.runtime.system_settings import STANDALONE_DATA_HOME + + +class StorageSession(StorageSessionBase): + def __init__(self, session_id, options=None): + super(StorageSession, self).__init__( + session_id=session_id, engine=StorageEngine.STANDALONE + ) + self._options = options if options else {} + self._session = Session( + session_id=self._session_id, data_dir=os.getenv("STANDALONE_DATA_HOME") or STANDALONE_DATA_HOME + ) + + def load( + self, + name, + namespace, + address: AddressABC, + store_type, + partitions, + options=None, + **kwargs, + ): + if isinstance(address, StandaloneAddress): + _table = self._session.load( + namespace=address.namespace, + name=address.name, + ) + + return StorageTable( + session=self._session, + table=_table, + name=name, + namespace=namespace, + address=address, + partitions=partitions, + store_type=store_type, + key_serdes_type=_table.key_serdes_type, + value_serdes_type=_table.value_serdes_type, + partitioner_type=_table.partitioner_type, + options=options, + ) + raise NotImplementedError( + f"address type {type(address)} not supported with standalone storage" + ) + + def table( + self, + name, + namespace, + address: AddressABC, + partitions, + key_serdes_type, + value_serdes_type, + partitioner_type, + store_type=None, + options=None, + **kwargs, + ): + if isinstance(address, StandaloneAddress): + _table = self._session.create_table( + namespace=address.namespace, + name=address.name, + partitions=partitions, + need_cleanup=store_type == StandaloneStoreType.ROLLPAIR_IN_MEMORY, + error_if_exist=False, + key_serdes_type=key_serdes_type, + value_serdes_type=value_serdes_type, + partitioner_type=partitioner_type, + ) + + return StorageTable( + session=self._session, + table=_table, + key_serdes_type=key_serdes_type, + value_serdes_type=value_serdes_type, + partitioner_type=partitioner_type, + name=name, + namespace=namespace, + address=address, + partitions=partitions, + store_type=store_type, + options=options, + ) + raise NotImplementedError( + f"address type {type(address)} not supported with standalone storage" + ) + + def cleanup(self, name, namespace): + self._session.cleanup(name=name, namespace=namespace) + + def stop(self): + self._session.stop() + + def kill(self): + self._session.kill() diff --git a/python/fate_flow/engine/storage/standalone/_standalone.py b/python/fate_flow/engine/storage/standalone/_standalone.py new file mode 100644 index 000000000..21f54c15f --- /dev/null +++ b/python/fate_flow/engine/storage/standalone/_standalone.py @@ -0,0 +1,1315 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import hashlib +import itertools +import logging +import logging.config +import os +import shutil +import signal +import threading +import time +import uuid +from concurrent.futures import ProcessPoolExecutor as Executor +from contextlib import ExitStack +from functools import partial +from heapq import heapify, heappop, heapreplace +from operator import is_not +from pathlib import Path +from typing import Callable, Any, Iterable, Optional +from typing import List, Tuple, Literal + +import cloudpickle as f_pickle +import lmdb + +PartyMeta = Tuple[Literal["guest", "host", "arbiter", "local"], str] + +logger = logging.getLogger(__name__) + + +def _watch_thread_react_to_parent_die(ppid, logger_config): + """ + this function is call when a process is created, and it will watch parent process and initialize loggers + Args: + ppid: parent process id + """ + + # watch parent process, if parent process is dead, then kill self + # the trick is to use os.kill(ppid, 0) to check if parent process is alive periodically + # and if parent process is dead, then kill self + # + # Note: this trick is modified from the answer by aaron: https://stackoverflow.com/a/71369760/14697733 + pid = os.getpid() + + def f(): + while True: + try: + os.kill(ppid, 0) + except OSError: + os.kill(pid, signal.SIGTERM) + time.sleep(1) + + thread = threading.Thread(target=f, daemon=True) + thread.start() + + # initialize loggers + if logger_config is not None: + logging.config.dictConfig(logger_config) + + +class BasicProcessPool: + def __init__(self, pool, log_level): + self._pool = pool + self._exception_tb = {} + self.log_level = log_level + + def submit(self, func, process_infos): + features = [] + outputs = {} + num_partitions = len(process_infos) + + for p, process_info in enumerate(process_infos): + features.append( + self._pool.submit( + BasicProcessPool._process_wrapper, + func, + process_info, + self.log_level, + ) + ) + + from concurrent.futures import wait, FIRST_COMPLETED + + not_done = features + while not_done: + done, not_done = wait(not_done, return_when=FIRST_COMPLETED) + for f in done: + partition_id, output, e = f.result() + if e is not None: + logger.error(f"partition {partition_id} exec failed: {e}") + raise RuntimeError(f"Partition {partition_id} exec failed: {e}") + else: + outputs[partition_id] = output + + outputs = [outputs[p] for p in range(num_partitions)] + return outputs + + @classmethod + def _process_wrapper(cls, do_func, process_info, log_level): + try: + if log_level is not None: + pass + output = do_func(process_info) + return process_info.partition_id, output, None + except Exception as e: + logger.error(f"exception in rank {process_info.partition_id}: {e}") + return process_info.partition_id, None, e + + def shutdown(self): + self._pool.shutdown() + + +# noinspection PyPep8Naming +class Table(object): + def __init__( + self, + session: "Session", + data_dir: str, + namespace: str, + name: str, + partitions, + key_serdes_type: int, + value_serdes_type: int, + partitioner_type: int, + need_cleanup=True, + ): + self._need_cleanup = need_cleanup + self._data_dir = data_dir + self._namespace = namespace + self._name = name + self._partitions = partitions + self._session = session + self._key_serdes_type = key_serdes_type + self._value_serdes_type = value_serdes_type + self._partitioner_type = partitioner_type + + @property + def num_partitions(self): + return self._partitions + + @property + def key_serdes_type(self): + return self._key_serdes_type + + @property + def value_serdes_type(self): + return self._value_serdes_type + + @property + def partitioner_type(self): + return self._partitioner_type + + @property + def partitions(self): + return self._partitions + + @property + def name(self): + return self._name + + @property + def namespace(self): + return self._namespace + + def __del__(self): + if self._need_cleanup: + try: + self.destroy() + except: + pass + + def __str__(self): + return f"" + + def __repr__(self): + return self.__str__() + + def destroy(self): + for p in range(self.num_partitions): + with self._get_env_for_partition(p, write=True) as env: + db = env.open_db() + with env.begin(write=True) as txn: + txn.drop(db) + _TableMetaManager.destroy_table(data_dir=self._data_dir, namespace=self._namespace, name=self._name) + + def take(self, num, **kwargs): + if num <= 0: + raise ValueError(f"{num} <= 0") + return list(itertools.islice(self.collect(**kwargs), num)) + + def count(self): + cnt = 0 + for p in range(self.num_partitions): + with self._get_env_for_partition(p) as env: + cnt += env.stat()["entries"] + return cnt + + # noinspection PyUnusedLocal + def collect(self, **kwargs): + iterators = [] + with ExitStack() as s: + for p in range(self.num_partitions): + env = s.enter_context(self._get_env_for_partition(p)) + txn = s.enter_context(env.begin()) + iterators.append(s.enter_context(txn.cursor())) + + # Merge sorted + entries = [] + for _id, it in enumerate(iterators): + if it.next(): + key, value = it.item() + entries.append([key, value, _id, it]) + heapify(entries) + while entries: + key, value, _, it = entry = entries[0] + yield key, value + if it.next(): + entry[0], entry[1] = it.item() + heapreplace(entries, entry) + else: + _, _, _, it = heappop(entries) + + def reduce(self, func): + return self._session.submit_reduce( + func, + data_dir=self._data_dir, + num_partitions=self.num_partitions, + name=self._name, + namespace=self._namespace, + ) + + def binary_sorted_map_partitions_with_index( + self, + other: "Table", + binary_map_partitions_with_index_op: Callable[[int, Iterable, Iterable], Iterable], + key_serdes_type, + partitioner_type, + output_value_serdes_type, + need_cleanup=True, + output_name=None, + output_namespace=None, + output_data_dir=None, + ): + if output_data_dir is None: + output_data_dir = self._data_dir + if output_name is None: + output_name = str(uuid.uuid1()) + if output_namespace is None: + output_namespace = self._namespace + + self._session._submit_sorted_binary_map_partitions_with_index( + func=binary_map_partitions_with_index_op, + do_func=_do_binary_sorted_map_with_index, + num_partitions=self.num_partitions, + first_input_data_dir=self._data_dir, + first_input_name=self._name, + first_input_namespace=self._namespace, + second_input_data_dir=other._data_dir, + second_input_name=other._name, + second_input_namespace=other._namespace, + output_data_dir=output_data_dir, + output_name=output_name, + output_namespace=output_namespace, + ) + return _create_table( + session=self._session, + data_dir=self._data_dir, + name=output_name, + namespace=output_namespace, + partitions=self.num_partitions, + need_cleanup=need_cleanup, + key_serdes_type=key_serdes_type, + value_serdes_type=output_value_serdes_type, + partitioner_type=partitioner_type, + ) + + def map_reduce_partitions_with_index( + self, + map_partition_op: Callable[[int, Iterable], Iterable], + reduce_partition_op: Optional[Callable[[Any, Any], Any]], + output_partitioner: Optional[Callable[[bytes, int], int]], + shuffle, + output_key_serdes_type, + output_value_serdes_type, + output_partitioner_type, + output_num_partitions, + need_cleanup=True, + output_name=None, + output_namespace=None, + output_data_dir=None, + ): + if output_data_dir is None: + output_data_dir = self._data_dir + if output_name is None: + output_name = str(uuid.uuid1()) + if output_namespace is None: + output_namespace = self._namespace + if not shuffle: + assert output_num_partitions == self.num_partitions and output_partitioner_type == self.partitioner_type + # noinspection PyProtectedMember + self._session._submit_map_reduce_partitions_with_index( + _do_mrwi_no_shuffle, + mapper=map_partition_op, + reducer=reduce_partition_op, + input_num_partitions=self.num_partitions, + input_data_dir=self._data_dir, + input_name=self._name, + input_namespace=self._namespace, + output_num_partitions=output_num_partitions, + output_data_dir=output_data_dir, + output_name=output_name, + output_namespace=output_namespace, + output_partitioner=output_partitioner, + ) + return _create_table( + session=self._session, + data_dir=output_data_dir, + name=output_name, + namespace=output_namespace, + partitions=output_num_partitions, + need_cleanup=need_cleanup, + key_serdes_type=output_key_serdes_type, + value_serdes_type=output_value_serdes_type, + partitioner_type=output_partitioner_type, + ) + + if reduce_partition_op is None: + # noinspection PyProtectedMember + self._session._submit_map_reduce_partitions_with_index( + _do_mrwi_shuffle_no_reduce, + map_partition_op, + reduce_partition_op, + input_data_dir=self._data_dir, + input_num_partitions=self.num_partitions, + input_name=self._name, + input_namespace=self._namespace, + output_data_dir=output_data_dir, + output_num_partitions=output_num_partitions, + output_name=output_name, + output_namespace=output_namespace, + output_partitioner=output_partitioner, + ) + return _create_table( + session=self._session, + data_dir=output_data_dir, + name=output_name, + namespace=output_namespace, + partitions=output_num_partitions, + need_cleanup=need_cleanup, + key_serdes_type=output_key_serdes_type, + value_serdes_type=output_value_serdes_type, + partitioner_type=output_partitioner_type, + ) + + # Step 1: do map and write intermediate results to cache table + # noinspection PyProtectedMember + intermediate_name = str(uuid.uuid1()) + intermediate_namespace = self._namespace + intermediate_data_dir = self._data_dir + self._session._submit_map_reduce_partitions_with_index( + _do_mrwi_map_and_shuffle_write, + mapper=map_partition_op, + reducer=None, + input_data_dir=self._data_dir, + input_num_partitions=self.num_partitions, + input_name=self._name, + input_namespace=self._namespace, + output_data_dir=intermediate_data_dir, + output_num_partitions=output_num_partitions, + output_name=intermediate_name, + output_namespace=intermediate_namespace, + output_partitioner=output_partitioner, + ) + # Step 2: do shuffle read and reduce + # noinspection PyProtectedMember + self._session._submit_map_reduce_partitions_with_index( + _do_mrwi_shuffle_read_and_reduce, + mapper=None, + reducer=reduce_partition_op, + input_data_dir=intermediate_data_dir, + input_num_partitions=self.num_partitions, + input_name=intermediate_name, + input_namespace=intermediate_namespace, + output_data_dir=output_data_dir, + output_num_partitions=output_num_partitions, + output_name=output_name, + output_namespace=output_namespace, + ) + output = _create_table( + session=self._session, + data_dir=output_data_dir, + name=output_name, + namespace=output_namespace, + partitions=output_num_partitions, + need_cleanup=need_cleanup, + key_serdes_type=output_key_serdes_type, + value_serdes_type=output_value_serdes_type, + partitioner_type=output_partitioner_type, + ) + + # drop cache table + for p in range(self._partitions): + with _get_env_with_data_dir( + intermediate_data_dir, intermediate_namespace, intermediate_name, str(p), write=True + ) as env: + db = env.open_db() + with env.begin(write=True) as txn: + txn.drop(db) + + path = Path(self._data_dir).joinpath(intermediate_namespace, intermediate_name) + shutil.rmtree(path, ignore_errors=True) + return output + + def copy_as(self, name, namespace, need_cleanup=True): + return self.map_reduce_partitions_with_index( + map_partition_op=lambda i, x: x, + reduce_partition_op=None, + output_partitioner=None, + shuffle=False, + need_cleanup=need_cleanup, + output_name=name, + output_namespace=namespace, + output_key_serdes_type=self._key_serdes_type, + output_value_serdes_type=self._value_serdes_type, + output_partitioner_type=self._partitioner_type, + output_num_partitions=self.num_partitions, + ) + + def _get_env_for_partition(self, p: int, write=False): + return _get_env_with_data_dir(self._data_dir, self._namespace, self._name, str(p), write=write) + + def put(self, k_bytes: bytes, v_bytes: bytes, partitioner: Callable[[bytes, int], int] = None): + p = partitioner(k_bytes, self._partitions) + with self._get_env_for_partition(p, write=True) as env: + with env.begin(write=True) as txn: + return txn.put(k_bytes, v_bytes) + + def put_all(self, kv_list: Iterable[Tuple[bytes, bytes]], partitioner: Callable[[bytes, int], int]): + txn_map = {} + with ExitStack() as s: + for p in range(self._partitions): + env = s.enter_context(self._get_env_for_partition(p, write=True)) + txn_map[p] = env, env.begin(write=True) + try: + for k_bytes, v_bytes in kv_list: + p = partitioner(k_bytes, self._partitions) + if not txn_map[p][1].put(k_bytes, v_bytes): + break + except Exception as e: + for p, (env, txn) in txn_map.items(): + txn.abort() + raise e + else: + for p, (env, txn) in txn_map.items(): + txn.commit() + + def get(self, k_bytes: bytes, partitioner: Callable[[bytes, int], int]) -> bytes: + p = partitioner(k_bytes, self._partitions) + with self._get_env_for_partition(p) as env: + with env.begin(write=True) as txn: + return txn.get(k_bytes) + + def delete(self, k_bytes: bytes, partitioner: Callable[[bytes, int], int]): + p = partitioner(k_bytes, self._partitions) + with self._get_env_for_partition(p, write=True) as env: + with env.begin(write=True) as txn: + old_value_bytes = txn.get(k_bytes) + if txn.delete(k_bytes): + return old_value_bytes + return None + + +# noinspection PyMethodMayBeStatic +class Session(object): + def __init__( + self, + session_id, + data_dir: str, + max_workers=None, + logger_config=None, + executor_pool_cls=BasicProcessPool, + ): + self.session_id = session_id + self._data_dir = data_dir + self._max_workers = max_workers + if self._max_workers is None: + self._max_workers = os.cpu_count() + + self._enable_process_logger = True + if self._enable_process_logger: + log_level = logging.getLevelName(logger.getEffectiveLevel()) + else: + log_level = None + self._pool = executor_pool_cls( + pool=Executor( + max_workers=max_workers, + initializer=_watch_thread_react_to_parent_die, + initargs=( + os.getpid(), + logger_config, + ), + ), + log_level=log_level, + ) + + @property + def data_dir(self): + return self._data_dir + + @property + def max_workers(self): + return self._max_workers + + def __getstate__(self): + # session won't be pickled + pass + + def load(self, name, namespace): + return _load_table(session=self, data_dir=self._data_dir, name=name, namespace=namespace) + + def create_table( + self, + name, + namespace, + partitions, + need_cleanup, + error_if_exist, + key_serdes_type, + value_serdes_type, + partitioner_type, + ): + return _create_table( + session=self, + data_dir=self._data_dir, + name=name, + namespace=namespace, + partitions=partitions, + need_cleanup=need_cleanup, + error_if_exist=error_if_exist, + key_serdes_type=key_serdes_type, + value_serdes_type=value_serdes_type, + partitioner_type=partitioner_type, + ) + + # noinspection PyUnusedLocal + def parallelize( + self, + data: Iterable, + partition: int, + partitioner: Callable[[bytes, int], int], + key_serdes_type, + value_serdes_type, + partitioner_type, + ): + table = _create_table( + session=self, + data_dir=self._data_dir, + name=str(uuid.uuid1()), + namespace=self.session_id, + partitions=partition, + need_cleanup=True, + key_serdes_type=key_serdes_type, + value_serdes_type=value_serdes_type, + partitioner_type=partitioner_type, + ) + table.put_all(data, partitioner=partitioner) + return table + + def cleanup(self, name, namespace): + path = Path(self._data_dir) + if not path.is_dir(): + return + namespace_dir = path.joinpath(namespace) + if not namespace_dir.is_dir(): + return + if name == "*": + shutil.rmtree(namespace_dir, True) + return + for table in namespace_dir.glob(name): + shutil.rmtree(table, True) + + def stop(self): + self.cleanup(name="*", namespace=self.session_id) + self._pool.shutdown() + + def kill(self): + self.cleanup(name="*", namespace=self.session_id) + self._pool.shutdown() + + def submit_reduce(self, func, data_dir: str, num_partitions: int, name: str, namespace: str): + rs = self._pool.submit( + _do_reduce, + [ + _ReduceProcess(p, _TaskInputInfo(data_dir, namespace, name, num_partitions), _ReduceFunctorInfo(func)) + for p in range(num_partitions) + ], + ) + rs = [r for r in filter(partial(is_not, None), rs)] + if len(rs) <= 0: + return None + rtn = rs[0] + for r in rs[1:]: + rtn = func(rtn, r) + return rtn + + def _submit_map_reduce_partitions_with_index( + self, + _do_func, + mapper, + reducer, + input_data_dir: str, + input_num_partitions, + input_name, + input_namespace, + output_data_dir: str, + output_num_partitions, + output_name, + output_namespace, + output_partitioner=None, + ): + input_info = _TaskInputInfo(input_data_dir, input_namespace, input_name, input_num_partitions) + output_info = _TaskOutputInfo( + output_data_dir, output_namespace, output_name, output_num_partitions, partitioner=output_partitioner + ) + return self._submit_process( + _do_func, + [ + _MapReduceProcess( + partition_id=p, + input_info=input_info, + output_info=output_info, + operator_info=_MapReduceFunctorInfo(mapper=mapper, reducer=reducer), + ) + for p in range(max(input_num_partitions, output_num_partitions)) + ], + ) + + def _submit_sorted_binary_map_partitions_with_index( + self, + func, + do_func, + num_partitions: int, + first_input_data_dir: str, + first_input_name: str, + first_input_namespace: str, + second_input_data_dir: str, + second_input_name: str, + second_input_namespace: str, + output_data_dir: str, + output_name: str, + output_namespace: str, + ): + first_input_info = _TaskInputInfo( + first_input_data_dir, first_input_namespace, first_input_name, num_partitions + ) + second_input_info = _TaskInputInfo( + second_input_data_dir, second_input_namespace, second_input_name, num_partitions + ) + output_info = _TaskOutputInfo(output_data_dir, output_namespace, output_name, num_partitions, partitioner=None) + return self._submit_process( + do_func, + [ + _BinarySortedMapProcess( + partition_id=p, + first_input_info=first_input_info, + second_input_info=second_input_info, + output_info=output_info, + operator_info=_BinarySortedMapFunctorInfo(func), + ) + for p in range(num_partitions) + ], + ) + + def _submit_process(self, do_func, process_infos): + return self._pool.submit(do_func, process_infos) + + +class Federation(object): + def _federation_object_key(self, name: str, tag: str, s_party: Tuple[str, str], d_party: Tuple[str, str]) -> bytes: + return f"{self._session_id}-{name}-{tag}-{s_party[0]}-{s_party[1]}-{d_party[0]}-{d_party[1]}".encode("utf-8") + + def __init__(self, session: Session, data_dir: str, session_id: str, party: Tuple[str, str]): + self._session = session + self._data_dir = data_dir + self._session_id = session_id + self._party = party + self._other_status_tables = {} + self._other_object_tables = {} + self._federation_status_table_cache = None + self._federation_object_table_cache = None + + self._meta = _FederationMetaManager(session_id=session_id, data_dir=data_dir, party=party) + + @classmethod + def create(cls, session: Session, session_id: str, party: Tuple[str, str]): + federation = cls(session, session.data_dir, session_id, party) + return federation + + def destroy(self): + self._session.cleanup(namespace=self._session_id, name="*") + + def push_table(self, table, name: str, tag: str, parties: List[PartyMeta]): + for party in parties: + _tagged_key = self._federation_object_key(name, tag, self._party, party) + saved_name = str(uuid.uuid1()) + _table = table.copy_as(name=saved_name, namespace=table.namespace, need_cleanup=False) + self._meta.set_status(party, _tagged_key, _serialize_tuple_of_str(_table.name, _table.namespace)) + + def push_bytes(self, v: bytes, name: str, tag: str, parties: List[PartyMeta]): + for party in parties: + _tagged_key = self._federation_object_key(name, tag, self._party, party) + self._meta.set_object(party, _tagged_key, v) + self._meta.set_status(party, _tagged_key, _tagged_key) + + def pull_table(self, name: str, tag: str, parties: List[PartyMeta]) -> List[Table]: + results: List[bytes] = [] + for party in parties: + _tagged_key = self._federation_object_key(name, tag, party, self._party) + + results.append(self._meta.wait_status_set(_tagged_key)) + + rtn = [] + for r in results: + name, namespace = _deserialize_tuple_of_str(self._meta.get_status(r)) + table: Table = _load_table( + session=self._session, data_dir=self._data_dir, name=name, namespace=namespace, need_cleanup=True + ) + rtn.append(table) + self._meta.ack_status(r) + return rtn + + def pull_bytes(self, name: str, tag: str, parties: List[PartyMeta]) -> List[bytes]: + results = [] + for party in parties: + _tagged_key = self._federation_object_key(name, tag, party, self._party) + results.append(self._meta.wait_status_set(_tagged_key)) + + rtn = [] + for r in results: + obj = self._meta.get_object(r) + if obj is None: + raise EnvironmentError(f"object not found: {r}") + rtn.append(obj) + self._meta.ack_object(r) + self._meta.ack_status(r) + return rtn + + +def _create_table( + session: "Session", + data_dir: str, + name: str, + namespace: str, + partitions: int, + key_serdes_type: int, + value_serdes_type: int, + partitioner_type: int, + need_cleanup=True, + error_if_exist=False, +): + assert isinstance(name, str) + assert isinstance(namespace, str) + assert isinstance(partitions, int) + if (exist_partitions := _TableMetaManager.get_table_meta(data_dir, namespace, name)) is None: + _TableMetaManager.add_table_meta( + data_dir, namespace, name, partitions, key_serdes_type, value_serdes_type, partitioner_type + ) + else: + if error_if_exist: + raise RuntimeError(f"table already exist: name={name}, namespace={namespace}") + partitions = exist_partitions + + return Table( + session=session, + data_dir=data_dir, + namespace=namespace, + name=name, + partitions=partitions, + key_serdes_type=key_serdes_type, + value_serdes_type=value_serdes_type, + partitioner_type=partitioner_type, + need_cleanup=need_cleanup, + ) + + +def _load_table(session, data_dir: str, name: str, namespace: str, need_cleanup=False): + table_meta = _TableMetaManager.get_table_meta(data_dir, namespace, name) + if table_meta is None: + raise RuntimeError(f"table not exist: name={name}, namespace={namespace}") + return Table( + session=session, + data_dir=data_dir, + namespace=namespace, + name=name, + need_cleanup=need_cleanup, + partitions=table_meta.num_partitions, + key_serdes_type=table_meta.key_serdes_type, + value_serdes_type=table_meta.value_serdes_type, + partitioner_type=table_meta.partitioner_type, + ) + + +class _TaskInputInfo: + def __init__(self, data_dir: str, namespace: str, name: str, num_partitions: int): + self.data_dir = data_dir + self.namespace = namespace + self.name = name + self.num_partitions = num_partitions + + def get_env(self, pid, write=False): + return _get_env_with_data_dir(self.data_dir, self.namespace, self.name, str(pid), write=write) + + +class _TaskOutputInfo: + def __init__(self, data_dir: str, namespace: str, name: str, num_partitions: int, partitioner): + self.data_dir = data_dir + self.namespace = namespace + self.name = name + self.num_partitions = num_partitions + self.partitioner = partitioner + + def get_env(self, pid, write=True): + return _get_env_with_data_dir(self.data_dir, self.namespace, self.name, str(pid), write=write) + + def get_partition_id(self, key): + if self.partitioner is None: + raise RuntimeError("partitioner is None") + return self.partitioner(key, self.num_partitions) + + +class _MapReduceFunctorInfo: + def __init__(self, mapper, reducer): + if mapper is not None: + self.mapper_bytes = f_pickle.dumps(mapper) + else: + self.mapper_bytes = None + if reducer is not None: + self.reducer_bytes = f_pickle.dumps(reducer) + else: + self.reducer_bytes = None + + def get_mapper(self): + if self.mapper_bytes is None: + raise RuntimeError("mapper is None") + return f_pickle.loads(self.mapper_bytes) + + def get_reducer(self): + if self.reducer_bytes is None: + raise RuntimeError("reducer is None") + return f_pickle.loads(self.reducer_bytes) + + +class _BinarySortedMapFunctorInfo: + def __init__(self, mapper): + if mapper is not None: + self.mapper_bytes = f_pickle.dumps(mapper) + else: + self.mapper_bytes = None + + def get_mapper(self): + if self.mapper_bytes is None: + raise RuntimeError("mapper is None") + return f_pickle.loads(self.mapper_bytes) + + +class _ReduceFunctorInfo: + def __init__(self, reducer): + if reducer is not None: + self.reducer_bytes = f_pickle.dumps(reducer) + else: + self.reducer_bytes = None + + def get_reducer(self): + if self.reducer_bytes is None: + raise RuntimeError("reducer is None") + return f_pickle.loads(self.reducer_bytes) + + +class _ReduceProcess: + def __init__( + self, + partition_id: int, + input_info: _TaskInputInfo, + operator_info: _ReduceFunctorInfo, + ): + self.partition_id = partition_id + self.input_info = input_info + self.operator_info = operator_info + + def as_input_env(self, pid, write=False): + return self.input_info.get_env(pid, write=write) + + def input_cursor(self, stack: ExitStack): + return stack.enter_context(stack.enter_context(self.as_input_env(self.partition_id).begin()).cursor()) + + def get_reducer(self): + return self.operator_info.get_reducer() + + +class _MapReduceProcess: + def __init__( + self, + partition_id: int, + input_info: _TaskInputInfo, + output_info: _TaskOutputInfo, + operator_info: _MapReduceFunctorInfo, + ): + self.partition_id = partition_id + self.input_info = input_info + self.output_info = output_info + self.operator_info = operator_info + + def get_input_partition_num(self): + return self.input_info.num_partitions + + def get_output_partition_num(self): + return self.output_info.num_partitions + + def get_input_env(self, pid, write=False): + return self.input_info.get_env(pid, write=write) + + def get_output_env(self, pid, write=True): + return self.output_info.get_env(pid, write=write) + + def get_input_cursor(self, stack: ExitStack, pid=None): + if pid is None: + pid = self.partition_id + if isinstance(pid, int) and pid >= self.input_info.num_partitions: + raise RuntimeError(f"pid {pid} >= input_info.num_partitions {self.input_info.num_partitions}") + return stack.enter_context( + stack.enter_context(stack.enter_context(self.get_input_env(pid, write=False)).begin(write=False)).cursor() + ) + + def has_partition(self, pid): + return pid < self.input_info.num_partitions + + def get_output_transaction(self, pid, stack: ExitStack): + return stack.enter_context(stack.enter_context(self.get_output_env(pid, write=True)).begin(write=True)) + + def get_output_partition_id(self, key: bytes): + return self.output_info.get_partition_id(key) + + def get_mapper(self): + return self.operator_info.get_mapper() + + def get_reducer(self): + return self.operator_info.get_reducer() + + +class _BinarySortedMapProcess: + def __init__( + self, + partition_id, + first_input_info: _TaskInputInfo, + second_input_info: _TaskInputInfo, + output_info: _TaskOutputInfo, + operator_info: _BinarySortedMapFunctorInfo, + ): + self.partition_id = partition_id + self.first_input = first_input_info + self.second_input = second_input_info + self.output_info = output_info + self.operator_info = operator_info + + def get_input_partition_num(self): + return self.first_input.num_partitions + + def get_output_partition_num(self): + return self.output_info.num_partitions + + def get_first_input_env(self, pid, write=False): + return self.first_input.get_env(pid, write=write) + + def get_second_input_env(self, pid, write=False): + return self.second_input.get_env(pid, write=write) + + def get_output_env(self, pid, write=True): + return self.output_info.get_env(pid, write=write) + + def get_first_input_cursor(self, stack: ExitStack, pid=None): + if pid is None: + pid = self.partition_id + return stack.enter_context( + stack.enter_context( + stack.enter_context(self.get_first_input_env(pid, write=False)).begin(write=False) + ).cursor() + ) + + def get_second_input_cursor(self, stack: ExitStack, pid=None): + if pid is None: + pid = self.partition_id + return stack.enter_context( + stack.enter_context( + stack.enter_context(self.get_second_input_env(pid, write=False)).begin(write=False) + ).cursor() + ) + + def get_output_transaction(self, pid, stack: ExitStack): + return stack.enter_context(stack.enter_context(self.get_output_env(pid, write=True)).begin(write=True)) + + def get_output_partition_id(self, key: bytes): + return self.output_info.get_partition_id(key) + + def get_func(self): + return self.operator_info.get_mapper() + + +def _get_env_with_data_dir(data_dir: str, *args, write=False): + _path = Path(data_dir).joinpath(*args) + return _open_env(_path, write=write) + + +def _open_env(path, write=False): + path.mkdir(parents=True, exist_ok=True) + + t = 0 + while t < 100: + try: + env = lmdb.open( + path.as_posix(), + create=True, + max_dbs=1, + max_readers=1024, + lock=write, + sync=True, + map_size=10_737_418_240, + ) + return env + except lmdb.Error as e: + if "No such file or directory" in e.args[0]: + time.sleep(0.001) + t += 1 + else: + raise e + raise lmdb.Error(f"No such file or directory: {path}, with {t} times retry") + + +def _generator_from_cursor(cursor): + for k, v in cursor: + yield k, v + + +def _do_mrwi_no_shuffle(p: _MapReduceProcess): + rtn = p.output_info + with ExitStack() as s: + dst_txn = p.get_output_transaction(p.partition_id, s) + cursor = p.get_input_cursor(s) + v = p.get_mapper()(p.partition_id, _generator_from_cursor(cursor)) + for k1, v1 in v: + dst_txn.put(k1, v1) + return rtn + + +def _do_mrwi_shuffle_no_reduce(p: _MapReduceProcess): + rtn = p.output_info + if p.has_partition(p.partition_id): + with ExitStack() as s: + cursor = p.get_input_cursor(s) + txn_map = {} + for output_partition_id in range(p.get_output_partition_num()): + txn_map[output_partition_id] = p.get_output_transaction(output_partition_id, s) + output_kv_iter = p.get_mapper()(p.partition_id, _generator_from_cursor(cursor)) + for k_bytes, v_bytes in output_kv_iter: + partition_id = p.get_output_partition_id(k_bytes) + txn_map[partition_id].put(k_bytes, v_bytes) + return rtn + + +def _do_binary_sorted_map_with_index(p: _BinarySortedMapProcess): + rtn = p.output_info + with ExitStack() as s: + first_cursor = p.get_first_input_cursor(s) + second_cursor = p.get_second_input_cursor(s) + dst_txn = p.get_output_transaction(p.partition_id, s) + output_kv_iter = p.get_func()( + p.partition_id, _generator_from_cursor(first_cursor), _generator_from_cursor(second_cursor) + ) + for k_bytes, v_bytes in output_kv_iter: + dst_txn.put(k_bytes, v_bytes) + return rtn + + +def _serialize_shuffle_write_key(iteration_index: int, k_bytes: bytes) -> bytes: + iteration_bytes = iteration_index.to_bytes(4, "big") # 4 bytes for the iteration index + serialized_key = iteration_bytes + k_bytes + + return serialized_key + + +def _deserialize_shuffle_write_key(serialized_key: bytes) -> (int, int, bytes): + iteration_bytes = serialized_key[:4] + k_bytes = serialized_key[4:] + iteration_index = int.from_bytes(iteration_bytes, "big") + return iteration_index, k_bytes + + +def _get_shuffle_partition_id(shuffle_source_partition_id: int, shuffle_destination_partition_id: int) -> str: + return f"{shuffle_source_partition_id}_{shuffle_destination_partition_id}" + + +def _do_mrwi_map_and_shuffle_write(p: _MapReduceProcess): + rtn = p.output_info + if p.has_partition(p.partition_id): + with ExitStack() as s: + cursor = p.get_input_cursor(s) + shuffle_write_txn_map = {} + for output_partition_id in range(p.get_output_partition_num()): + shuffle_partition_id = _get_shuffle_partition_id(p.partition_id, output_partition_id) + shuffle_write_txn_map[output_partition_id] = p.get_output_transaction(shuffle_partition_id, s) + + output_kv_iter = p.get_mapper()(p.partition_id, _generator_from_cursor(cursor)) + for index, (k_bytes, v_bytes) in enumerate(output_kv_iter): + shuffle_write_txn_map[p.get_output_partition_id(k_bytes)].put( + _serialize_shuffle_write_key(index, k_bytes), v_bytes, overwrite=False + ) + return rtn + + +def _do_mrwi_shuffle_read_and_reduce(p: _MapReduceProcess): + rtn = p.output_info + reducer = p.get_reducer() + with ExitStack() as s: + dst_txn = p.get_output_transaction(p.partition_id, s) + for input_partition_id in range(p.get_input_partition_num()): + for k_bytes, v_bytes in p.get_input_cursor( + s, pid=_get_shuffle_partition_id(input_partition_id, p.partition_id) + ): + _, key = _deserialize_shuffle_write_key(k_bytes) + if (old := dst_txn.get(key)) is None: + dst_txn.put(key, v_bytes) + else: + dst_txn.put(key, reducer(old, v_bytes)) + return rtn + + +def _do_reduce(p: _ReduceProcess): + value = None + with ExitStack() as s: + cursor = p.input_cursor(s) + for _, v_bytes in cursor: + if value is None: + value = v_bytes + else: + value = p.get_reducer()(value, v_bytes) + return value + + +class _FederationMetaManager: + STATUS_TABLE_NAME_PREFIX = "__federation_status__" + OBJECT_TABLE_NAME_PREFIX = "__federation_object__" + + def __init__(self, data_dir: str, session_id, party: Tuple[str, str]) -> None: + self.session_id = session_id + self.party = party + self._data_dir = data_dir + self._env = {} + + def wait_status_set(self, key: bytes) -> bytes: + value = self.get_status(key) + while value is None: + time.sleep(0.001) + value = self.get_status(key) + return key + + def get_status(self, key: bytes): + return self._get(self._get_status_table_name(self.party), key) + + def set_status(self, party: Tuple[str, str], key: bytes, value: bytes): + return self._set(self._get_status_table_name(party), key, value) + + def ack_status(self, key: bytes): + return self._ack(self._get_status_table_name(self.party), key) + + def get_object(self, key: bytes): + return self._get(self._get_object_table_name(self.party), key) + + def set_object(self, party: Tuple[str, str], key: bytes, value: bytes): + return self._set(self._get_object_table_name(party), key, value) + + def ack_object(self, key: bytes): + return self._ack(self._get_object_table_name(self.party), key) + + def _get_status_table_name(self, party: Tuple[str, str]): + return f"{self.STATUS_TABLE_NAME_PREFIX}.{party[0]}_{party[1]}" + + def _get_object_table_name(self, party: Tuple[str, str]): + return f"{self.OBJECT_TABLE_NAME_PREFIX}.{party[0]}_{party[1]}" + + def _get_env(self, name): + if name not in self._env: + self._env[name] = _get_env_with_data_dir(self._data_dir, self.session_id, name, str(0), write=True) + return self._env[name] + + def _get(self, name: str, key: bytes) -> bytes: + env = self._get_env(name) + with env.begin(write=False) as txn: + return txn.get(key) + + def _set(self, name, key: bytes, value: bytes): + env = self._get_env(name) + with env.begin(write=True) as txn: + return txn.put(key, value) + + def _ack(self, name, key: bytes): + env = self._get_env(name) + with env.begin(write=True) as txn: + txn.delete(key) + + +def _hash_namespace_name_to_partition(namespace: str, name: str, partitions: int) -> Tuple[bytes, int]: + k_bytes = f"{name}.{namespace}".encode("utf-8") + partition_id = int.from_bytes(hashlib.sha256(k_bytes).digest(), "big") % partitions + return k_bytes, partition_id + + +class _TableMetaManager: + namespace = "__META__" + name = "fragments" + num_partitions = 11 + _env = {} + + @classmethod + def _get_or_create_meta_env(cls, data_dir: str, p): + if p not in cls._env: + cls._env[p] = _get_env_with_data_dir(data_dir, cls.namespace, cls.name, str(p), write=True) + return cls._env[p] + + @classmethod + def _get_meta_env(cls, data_dir: str, namespace: str, name: str): + k_bytes, p = _hash_namespace_name_to_partition(namespace, name, cls.num_partitions) + env = cls._get_or_create_meta_env(data_dir, p) + return k_bytes, env + + @classmethod + def add_table_meta( + cls, + data_dir: str, + namespace: str, + name: str, + num_partitions: int, + key_serdes_type: int, + value_serdes_type: int, + partitioner_type: int, + ): + k_bytes, env = cls._get_meta_env(data_dir, namespace, name) + meta = _TableMeta(num_partitions, key_serdes_type, value_serdes_type, partitioner_type) + with env.begin(write=True) as txn: + return txn.put(k_bytes, meta.serialize()) + + @classmethod + def get_table_meta(cls, data_dir: str, namespace: str, name: str) -> "_TableMeta": + k_bytes, env = cls._get_meta_env(data_dir, namespace, name) + with env.begin(write=False) as txn: + old_value_bytes = txn.get(k_bytes) + if old_value_bytes is not None: + old_value_bytes = _TableMeta.deserialize(old_value_bytes) + return old_value_bytes + + @classmethod + def destroy_table(cls, data_dir: str, namespace: str, name: str): + k_bytes, env = cls._get_meta_env(data_dir, namespace, name) + with env.begin(write=True) as txn: + txn.delete(k_bytes) + path = Path(data_dir).joinpath(namespace, name) + shutil.rmtree(path, ignore_errors=True) + + +class _TableMeta: + def __init__(self, num_partitions: int, key_serdes_type: int, value_serdes_type: int, partitioner_type: int): + self.num_partitions = num_partitions + self.key_serdes_type = key_serdes_type + self.value_serdes_type = value_serdes_type + self.partitioner_type = partitioner_type + + def serialize(self) -> bytes: + num_partitions_bytes = self.num_partitions.to_bytes(4, "big") + key_serdes_type_bytes = self.key_serdes_type.to_bytes(4, "big") + value_serdes_type_bytes = self.value_serdes_type.to_bytes(4, "big") + partitioner_type_bytes = self.partitioner_type.to_bytes(4, "big") + return num_partitions_bytes + key_serdes_type_bytes + value_serdes_type_bytes + partitioner_type_bytes + + @classmethod + def deserialize(cls, serialized_bytes: bytes) -> "_TableMeta": + num_partitions = int.from_bytes(serialized_bytes[:4], "big") + key_serdes_type = int.from_bytes(serialized_bytes[4:8], "big") + value_serdes_type = int.from_bytes(serialized_bytes[8:12], "big") + partitioner_type = int.from_bytes(serialized_bytes[12:16], "big") + return cls(num_partitions, key_serdes_type, value_serdes_type, partitioner_type) + + +def _serialize_tuple_of_str(name: str, namespace: str): + name_bytes = name.encode("utf-8") + namespace_bytes = namespace.encode("utf-8") + split_index_bytes = len(name_bytes).to_bytes(4, "big") + return split_index_bytes + name_bytes + namespace_bytes + + +def _deserialize_tuple_of_str(serialized_bytes: bytes): + split_index = int.from_bytes(serialized_bytes[:4], "big") + name = serialized_bytes[4 : 4 + split_index].decode("utf-8") + namespace = serialized_bytes[4 + split_index :].decode("utf-8") + return name, namespace diff --git a/python/fate_flow/engine/storage/standalone/_table.py b/python/fate_flow/engine/storage/standalone/_table.py new file mode 100644 index 000000000..5eac7dcfa --- /dev/null +++ b/python/fate_flow/engine/storage/standalone/_table.py @@ -0,0 +1,66 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Iterable + +from fate_flow.engine.storage import ( + StandaloneStoreType, + StorageEngine, + StorageTableBase, +) +from fate_flow.engine.storage.standalone._standalone import Session + + +class StorageTable(StorageTableBase): + def __init__( + self, + session: Session, + table, + key_serdes_type, + value_serdes_type, + partitioner_type, + address=None, + name: str = None, + namespace: str = None, + partitions: int = 1, + store_type: StandaloneStoreType = StandaloneStoreType.ROLLPAIR_LMDB, + options=None, + ): + super(StorageTable, self).__init__( + name=name, + namespace=namespace, + address=address, + partitions=partitions, + options=options, + engine=StorageEngine.STANDALONE, + key_serdes_type=key_serdes_type, + value_serdes_type=value_serdes_type, + partitioner_type=partitioner_type, + ) + self._store_type = store_type + self._session = session + self._table = table + + def _put_all(self, kv_list: Iterable, partitioner, **kwargs): + return self._table.put_all(kv_list, partitioner) + + def _collect(self, **kwargs): + return self._table.collect(**kwargs) + + def _count(self): + return self._table.count() + + def _destroy(self): + self._table.destroy() diff --git a/python/fate_flow/entity/__init__.py b/python/fate_flow/entity/__init__.py index 2282bccae..1bb3927e4 100644 --- a/python/fate_flow/entity/__init__.py +++ b/python/fate_flow/entity/__init__.py @@ -13,10 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from ._base import BaseEntity -from ._data_cache import DataCache -from ._component_provider import ComponentProvider -from ._job import JobConfigurationBase, JobConfiguration -from ._metric import MetricType, Metric, MetricMeta -from .types import RetCode -from ._run_parameters import RunParameters \ No newline at end of file + +from ._base import BaseEntity, BaseModel, CustomEnum + +__all__ = ["BaseEntity", "BaseModel", "CustomEnum"] diff --git a/python/fate_flow/entity/_base.py b/python/fate_flow/entity/_base.py index 928b1d2a7..e13d2b4e6 100644 --- a/python/fate_flow/entity/_base.py +++ b/python/fate_flow/entity/_base.py @@ -13,8 +13,41 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from fate_arch.common import BaseType +from enum import Enum + +from pydantic import BaseModel as Base + +from fate_flow.utils.base_utils import BaseType class BaseEntity(BaseType): pass + + +class BaseModel(Base): + def to_dict(self): + d = {} + for k, v in self.__dict__.items(): + d[k] = v + return d + + def __str__(self): + return str(self.to_dict()) + + +class CustomEnum(Enum): + @classmethod + def valid(cls, value): + try: + cls(value) + return True + except: + return False + + @classmethod + def values(cls): + return [member.value for member in cls.__members__.values()] + + @classmethod + def names(cls): + return [member.name for member in cls.__members__.values()] \ No newline at end of file diff --git a/python/fate_flow/entity/_data_cache.py b/python/fate_flow/entity/_data_cache.py deleted file mode 100644 index b79844b3c..000000000 --- a/python/fate_flow/entity/_data_cache.py +++ /dev/null @@ -1,82 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import typing -from ._base import BaseEntity -from fate_arch.common import DTable - - -class DataCache(BaseEntity): - def __init__(self, name: str, key: str = None, data: typing.Dict[str, DTable] = None, meta: dict = None, job_id: str = None, component_name: str = None, task_id: str = None, task_version: int = None): - self._name: str = name - self._key: str = key - self._data: typing.Dict[str, DTable] = data if data else {} - self._meta: dict = meta - self._job_id = job_id - self._component_name = component_name - self._task_id: str = task_id - self._task_version: int = task_version - - @property - def name(self): - return self._name - - @property - def key(self): - return self._key - - @key.setter - def key(self, key: str): - self._key = key - - @property - def data(self): - return self._data - - @property - def meta(self): - return self._meta - - @property - def job_id(self): - return self._job_id - - @job_id.setter - def job_id(self, job_id: str): - self._job_id = job_id - - @property - def component_name(self): - return self._component_name - - @component_name.setter - def component_name(self, component_name: str): - self._component_name = component_name - - @property - def task_id(self): - return self._task_id - - @task_id.setter - def task_id(self, task_id: str): - self._task_id = task_id - - @property - def task_version(self): - return self._task_version - - @task_version.setter - def task_version(self, task_version: int): - self._task_version = task_version diff --git a/python/fate_flow/entity/_job.py b/python/fate_flow/entity/_job.py deleted file mode 100644 index 243cdb76a..000000000 --- a/python/fate_flow/entity/_job.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from ._base import BaseEntity - - -class JobConfigurationBase(BaseEntity): - def __init__(self, dsl=None, runtime_conf=None, **kwargs): - self._dsl = dsl if dsl else kwargs.get("job_dsl") - self._runtime_conf = runtime_conf if runtime_conf else kwargs.get("job_runtime_conf") - - @property - def dsl(self): - return self._dsl - - @property - def runtime_conf(self): - return self._runtime_conf - - -class JobConfiguration(JobConfigurationBase): - def __init__(self, dsl, runtime_conf, runtime_conf_on_party, train_runtime_conf, **kwargs): - super().__init__(dsl, runtime_conf, **kwargs) - self._runtime_conf_on_party = runtime_conf_on_party - self._train_runtime_conf = train_runtime_conf - - @property - def runtime_conf_on_party(self): - return self._runtime_conf_on_party - - @property - def train_runtime_conf(self): - return self._train_runtime_conf - diff --git a/python/fate_flow/entity/_metric.py b/python/fate_flow/entity/_metric.py deleted file mode 100644 index 8daa78cdb..000000000 --- a/python/fate_flow/entity/_metric.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import typing -from enum import Enum -from fate_flow.entity import BaseEntity - - -class MetricType(Enum): - LOSS = 'LOSS' - DOWNLOAD = 'DOWNLOAD' - CACHE_INFO = 'cache_info' - CHECKPOINT_INFO = 'checkpoint_info' - UPLOAD = 'UPLOAD' - COMPONENT_MODEL_INFO = 'component_model_info' - - -class Metric(BaseEntity): - def __init__(self, key, value: float, timestamp: float = None): - self.key = key - self.value = value - self.timestamp = timestamp - - @classmethod - def from_dict(cls, d: dict): - return Metric(d.get("key"), d.get("value"), d.get("timestamp")) - - -class MetricMeta(BaseEntity): - def __init__(self, name: str, metric_type: typing.Union[MetricType, str], extra_metas: dict = None): - self.name = name - self.metric_type = metric_type - self.metas = {} - if extra_metas: - self.metas.update(extra_metas) - self.metas['name'] = name - self.metas['metric_type'] = metric_type - - def update_metas(self, metas: dict): - self.metas.update(metas) - - def to_dict(self): - return self.metas - - @classmethod - def from_dict(cls, d: dict): - metas = d.get("metas", {}) - if d.get("extra_metas"): - metas.update(d["extra_metas"]) - return MetricMeta(d.get("name"), d.get("metric_type"), extra_metas=metas) - - diff --git a/python/fate_flow/entity/_run_parameters.py b/python/fate_flow/entity/_run_parameters.py deleted file mode 100644 index a3bcfe7a9..000000000 --- a/python/fate_flow/entity/_run_parameters.py +++ /dev/null @@ -1,85 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from copy import deepcopy - -from ._base import BaseEntity -from ..utils.runtime_conf_parse_util import RuntimeConfParserUtil - - -class RunParameters(BaseEntity): - def __init__(self, **kwargs): - self.job_type = "train" - self.inheritance_info = {} # job_id, component_list - self.computing_engine = None - self.federation_engine = None - self.storage_engine = None - self.engines_address = {} - self.federated_mode = None - self.federation_info = None - self.task_cores = None - self.task_parallelism = None - self.computing_partitions = None - self.federated_status_collect_type = None - self.federated_data_exchange_type = None # not use in v1.5.0 - self.model_id = None - self.model_version = None - self.dsl_version = None - self.auto_retries = None - self.auto_retry_delay = None - self.timeout = None - self.eggroll_run = {} - self.spark_run = {} - self.rabbitmq_run = {} - self.pulsar_run = {} - self.adaptation_parameters = {} - self.assistant_role = None - self.map_table_name = None - self.map_namespace = None - self.task_conf = {} - self.roles = {} - self.role_parameters = {} - for k, v in kwargs.items(): - if hasattr(self, k): - setattr(self, k, v) - - def role_parameter(self, parameter_name, role, party_id): - if role == "local" or int(party_id) == 0: - return {} - if not self.roles: - # compatible with previous versions - return getattr(self, parameter_name) - index = [str(_p) for _p in self.roles.get(role)].index(str(party_id)) - _conf = deepcopy(getattr(self, parameter_name)) - _role = self.role_parameters.get(role, {}).get(str(index), {}).get(parameter_name, {}) - if isinstance(_conf, dict): - # dict - _conf = RuntimeConfParserUtil.merge_dict(_conf, _role) - else: - # int, str, etc. - _conf = _role if _role else _conf - return _conf - - - def to_dict(self): - d = {} - for k, v in self.__dict__.items(): - if v is None: - continue - d[k] = v - return d - - def __str__(self): - return str(self.to_dict()) diff --git a/python/fate_flow/entity/code/__init__.py b/python/fate_flow/entity/code/__init__.py new file mode 100644 index 000000000..868f7c581 --- /dev/null +++ b/python/fate_flow/entity/code/__init__.py @@ -0,0 +1,18 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from ._schedule import * +from ._api import * +from ._process import * diff --git a/python/fate_flow/entity/code/_api.py b/python/fate_flow/entity/code/_api.py new file mode 100644 index 000000000..960285208 --- /dev/null +++ b/python/fate_flow/entity/code/_api.py @@ -0,0 +1,73 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +class ReturnCode: + + class Base: + SUCCESS = 0 + + class Job: + PARAMS_ERROR = 1000 + NOT_FOUND = 1001 + CREATE_JOB_FAILED = 1002 + UPDATE_FAILED = 1003 + KILL_FAILED = 1004 + RESOURCE_EXCEPTION = 1005 + INHERITANCE_FAILED = 1006 + + class Task: + NOT_FOUND = 2000 + START_FAILED = 2001 + UPDATE_FAILED = 2002 + KILL_FAILED = 2003 + RESOURCE_EXCEPTION = 2004 + NO_FOUND_MODEL_OUTPUT = 2005 + TASK_RUN_FAILED = 2006 + COMPONENT_RUN_FAILED = 2007 + NO_FOUND_RUN_RESULT = 2008 + + class Site: + IS_STANDALONE = 3000 + + class Provider: + PARAMS_ERROR = 4000 + DEVICE_NOT_SUPPORTED = 4001 + + class API: + EXPIRED = 5000 + INVALID_PARAMETER = 5001 + NO_FOUND_APPID = 5002 + VERIFY_FAILED = 5003 + AUTHENTICATION_FAILED = 5004 + NO_PERMISSION = 5005 + PERMISSION_OPERATE_ERROR = 5006 + NO_FOUND_FILE = 5007 + COMPONENT_OUTPUT_EXCEPTION = 5008 + ROLE_TYPE_ERROR = 5009 + + class Server: + EXCEPTION = 6000 + FUNCTION_RESTRICTED = 6001 + RESPONSE_EXCEPTION = 6002 + NO_FOUND = 6003 + NO_FOUND_INSTANCE = 6004 + + class Table: + NO_FOUND = 7001 + EXISTS = 7002 + + class File: + FILE_NOT_FOUND = 8001 + FILE_EXISTS = 8002 diff --git a/python/fate_flow/entity/code/_process.py b/python/fate_flow/entity/code/_process.py new file mode 100644 index 000000000..15ace6ed0 --- /dev/null +++ b/python/fate_flow/entity/code/_process.py @@ -0,0 +1,24 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from enum import IntEnum + +from fate_flow.entity import CustomEnum + + +class KillProcessRetCode(IntEnum, CustomEnum): + KILLED = 0 + NOT_FOUND = 1 + ERROR_PID = 2 diff --git a/python/fate_flow/entity/code/_schedule.py b/python/fate_flow/entity/code/_schedule.py new file mode 100644 index 000000000..38300bd4c --- /dev/null +++ b/python/fate_flow/entity/code/_schedule.py @@ -0,0 +1,27 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +class SchedulingStatusCode(object): + SUCCESS = 0 + NO_RESOURCE = 1 + PASS = 1 + NO_NEXT = 2 + HAVE_NEXT = 3 + FAILED = 4 + + +class FederatedSchedulingStatusCode(object): + SUCCESS = 0 + FAILED = 1 diff --git a/python/fate_flow/detection/__init__.py b/python/fate_flow/entity/spec/__init__.py similarity index 100% rename from python/fate_flow/detection/__init__.py rename to python/fate_flow/entity/spec/__init__.py diff --git a/python/fate_flow/entity/spec/dag/__init__.py b/python/fate_flow/entity/spec/dag/__init__.py new file mode 100644 index 000000000..ddab5704b --- /dev/null +++ b/python/fate_flow/entity/spec/dag/__init__.py @@ -0,0 +1,32 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.entity.spec.dag._output import ComponentOutputMeta, MetricData, OutputArtifactType, OutputArtifactSpec, \ + OutputArtifacts, IOMeta +from fate_flow.entity.spec.dag._party import PartySpec +from fate_flow.entity.spec.dag._job import DAGSchema, DAGSpec, JobConfSpec, TaskConfSpec, TaskSpec, PartyTaskSpec, \ + InheritConfSpec, PartyTaskRefSpec +from fate_flow.entity.spec.dag._task import TaskConfigSpec, PreTaskConfigSpec, TaskRuntimeConfSpec, \ + TaskCleanupConfigSpec +from fate_flow.entity.spec.dag._artifact import RuntimeTaskOutputChannelSpec, DataWarehouseChannelSpec, \ + ModelWarehouseChannelSpec, RuntimeInputArtifacts, FlowRuntimeInputArtifacts,\ + ArtifactInputApplySpec, Metadata, RuntimeTaskOutputChannelSpec, InputArtifactSpec, \ + ArtifactOutputApplySpec, ModelWarehouseChannelSpec, ArtifactOutputSpec, ArtifactSource +from fate_flow.entity.spec.dag._component import ComponentSpec, ComponentIOArtifactsTypeSpec, ComponentSpecV1 +from fate_flow.entity.spec.dag._computing import EggrollComputingSpec, SparkComputingSpec, StandaloneComputingSpec +from fate_flow.entity.spec.dag._federation import StandaloneFederationSpec, RollSiteFederationSpec, OSXFederationSpec, \ + PulsarFederationSpec, RabbitMQFederationSpec +from fate_flow.entity.spec.dag._logger import FlowLogger +from fate_flow.entity.spec.dag._mlmd import MLMDSpec diff --git a/python/fate_flow/entity/spec/dag/_artifact.py b/python/fate_flow/entity/spec/dag/_artifact.py new file mode 100644 index 000000000..67e074ca0 --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_artifact.py @@ -0,0 +1,176 @@ +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +from typing import Optional, List, Literal, TypeVar, Dict, Union + +import pydantic + +# see https://www.rfc-editor.org/rfc/rfc3986#appendix-B +# scheme = $2 +# authority = $4 +# path = $5 +# query = $7 +# fragment = $9 +from ._party import PartySpec + +_uri_regex = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") + + +class ArtifactSource(pydantic.BaseModel): + task_id: str + party_task_id: str + task_name: str + component: str + output_artifact_key: str + output_index: Optional[int] = None + + +class Metadata(pydantic.BaseModel): + class DataOverview(pydantic.BaseModel): + count: Optional[int] = None + samples: Optional[List] = None + metadata: dict = pydantic.Field(default_factory=dict) + name: Optional[str] = None + namespace: Optional[str] = None + model_overview: Optional[dict] = {} + data_overview: Optional[DataOverview] + source: Optional[ArtifactSource] = None + model_key: Optional[str] + type_name: Optional[str] + index: Optional[Union[int, None]] = None + + class Config: + extra = "forbid" + + +class ArtifactInputApplySpec(pydantic.BaseModel): + uri: str + metadata: Metadata + type_name: Optional[str] = None + + def get_uri(self) -> "URI": + return URI.from_string(self.uri) + + +class ArtifactOutputApplySpec(pydantic.BaseModel): + uri: str + _is_template: Optional[bool] = None + type_name: Optional[str] = None + + def get_uri(self, index) -> "URI": + if self.is_template(): + return URI.from_string(self.uri.format(index=index)) + else: + if index != 0: + raise ValueError(f"index should be 0, but got {index}") + return URI.from_string(self.uri) + + def is_template(self) -> bool: + return "{index}" in self.uri + + def _check_is_template(self) -> bool: + return "{index}" in self.uri + + @pydantic.validator("uri") + def _check_uri(cls, v, values) -> str: + if not _uri_regex.match(v): + raise pydantic.ValidationError(f"`{v}` is not valid uri") + return v + + +class ArtifactOutputSpec(pydantic.BaseModel): + uri: str + metadata: Metadata + type_name: str + consumed: Optional[bool] = None + + +class URI: + def __init__( + self, + schema: str, + path: str, + query: Optional[str] = None, + fragment: Optional[str] = None, + authority: Optional[str] = None, + ): + self.schema = schema + self.path = path + self.query = query + self.fragment = fragment + self.authority = authority + + @classmethod + def from_string(cls, uri: str) -> "URI": + match = _uri_regex.fullmatch(uri) + if match is None: + raise ValueError(f"`{uri}` is not valid uri") + _, schema, _, authority, path, _, query, _, fragment = match.groups() + return URI(schema=schema, path=path, query=query, fragment=fragment, authority=authority) + + @classmethod + def load_uri(cls, engine, address): + pass + + +class RuntimeTaskOutputChannelSpec(pydantic.BaseModel): + producer_task: str + output_artifact_key: str + output_artifact_type_alias: Optional[str] + parties: Optional[List[PartySpec]] + + class Config: + extra = "forbid" + + +class DataWarehouseChannelSpec(pydantic.BaseModel): + job_id: Optional[str] + producer_task: Optional[str] + output_artifact_key: Optional[str] + namespace: Optional[str] + name: Optional[str] + dataset_id: Optional[str] + parties: Optional[List[PartySpec]] + + class Config: + extra = "forbid" + + +class ModelWarehouseChannelSpec(pydantic.BaseModel): + model_id: Optional[str] + model_version: Optional[str] + producer_task: str + output_artifact_key: str + parties: Optional[List[PartySpec]] + + class Config: + extra = "forbid" + + +InputArtifactSpec = TypeVar("InputArtifactSpec", + RuntimeTaskOutputChannelSpec, + ModelWarehouseChannelSpec, + DataWarehouseChannelSpec) + + +class RuntimeInputArtifacts(pydantic.BaseModel): + data: Optional[Dict[str, Dict[str, Union[List[InputArtifactSpec], InputArtifactSpec]]]] + model: Optional[Dict[str, Dict[str, Union[List[InputArtifactSpec], InputArtifactSpec]]]] + + +class FlowRuntimeInputArtifacts(pydantic.BaseModel): + data: Optional[Dict[str, Union[InputArtifactSpec, List[InputArtifactSpec]]]] + model: Optional[Dict[str, Union[InputArtifactSpec, List[InputArtifactSpec]]]] diff --git a/python/fate_flow/entity/spec/dag/_component.py b/python/fate_flow/entity/spec/dag/_component.py new file mode 100644 index 000000000..1d24d0a10 --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_component.py @@ -0,0 +1,100 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional, Dict, List, Union, Any, Literal +from pydantic import BaseModel + + +class ParameterSpec(BaseModel): + type: str + default: Optional[Any] + optional: bool + description: str = "" + type_meta: dict = {} + + +class ArtifactSpec(BaseModel): + types: List[str] + optional: bool + stages: Optional[List[str]] + roles: Optional[List[str]] + description: str = "" + is_multi: bool + + +class InputArtifactsSpec(BaseModel): + data: Dict[str, ArtifactSpec] + model: Dict[str, ArtifactSpec] + + +class OutputArtifactsSpec(BaseModel): + data: Dict[str, ArtifactSpec] + model: Dict[str, ArtifactSpec] + metric: Dict[str, ArtifactSpec] + + +class ComponentSpec(BaseModel): + name: str + description: str + provider: str + version: str + labels: List[str] = ["trainable"] + roles: List[str] + parameters: Dict[str, ParameterSpec] + input_artifacts: InputArtifactsSpec + output_artifacts: OutputArtifactsSpec + + +class RuntimeOutputChannelSpec(BaseModel): + producer_task: str + output_artifact_key: str + + +class RuntimeInputDefinition(BaseModel): + parameters: Optional[Dict[str, Any]] + artifacts: Optional[Dict[str, Dict[str, RuntimeOutputChannelSpec]]] + + +class ArtifactTypeSpec(BaseModel): + type_name: str + uri_types: List[str] + path_type: Literal["file", "directory", "distributed", "unresolved"] + + +class ComponentIOArtifactTypeSpec(BaseModel): + name: str + is_multi: bool + optional: bool + types: List[ArtifactTypeSpec] + + +class ComponentIOInputsArtifactsTypeSpec(BaseModel): + data: List[ComponentIOArtifactTypeSpec] + model: List[ComponentIOArtifactTypeSpec] + + +class ComponentIOOutputsArtifactsTypeSpec(BaseModel): + data: List[ComponentIOArtifactTypeSpec] + model: List[ComponentIOArtifactTypeSpec] + metric: List[ComponentIOArtifactTypeSpec] + + +class ComponentIOArtifactsTypeSpec(BaseModel): + inputs: ComponentIOInputsArtifactsTypeSpec + outputs: ComponentIOOutputsArtifactsTypeSpec + + +class ComponentSpecV1(BaseModel): + component: ComponentSpec + schema_version: str = "v1" diff --git a/python/fate_flow/entity/spec/dag/_computing.py b/python/fate_flow/entity/spec/dag/_computing.py new file mode 100644 index 000000000..64934c644 --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_computing.py @@ -0,0 +1,54 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Literal, TypeVar + +import pydantic +from pydantic import typing + + +class StandaloneComputingSpec(pydantic.BaseModel): + class MetadataSpec(pydantic.BaseModel): + computing_id: str + options: dict = {} + + type: Literal["standalone"] + metadata: MetadataSpec + + +class EggrollComputingSpec(pydantic.BaseModel): + class MetadataSpec(pydantic.BaseModel): + computing_id: str + host: typing.Optional[str] = None + port: typing.Optional[int] = None + config_options: typing.Optional[dict] = None + config_properties_file: typing.Optional[str] = None + options: dict = {} + + type: Literal["eggroll"] + metadata: MetadataSpec + + +class SparkComputingSpec(pydantic.BaseModel): + class MetadataSpec(pydantic.BaseModel): + computing_id: str + options: dict = {} + + type: Literal["spark"] + metadata: MetadataSpec + + +class CustomComputingSpec(pydantic.BaseModel): + type: Literal["custom"] + metadata: dict diff --git a/python/fate_flow/entity/spec/dag/_device.py b/python/fate_flow/entity/spec/dag/_device.py new file mode 100644 index 000000000..447e63d72 --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_device.py @@ -0,0 +1,28 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Literal + +import pydantic +from pydantic import typing + + +class CPUSpec(pydantic.BaseModel): + type: Literal["CPU"] + metadata: dict = {} + + +class GPUSpec(pydantic.BaseModel): + type: Literal["GPU"] + metadata: dict = {} diff --git a/python/fate_flow/entity/spec/dag/_federation.py b/python/fate_flow/entity/spec/dag/_federation.py new file mode 100644 index 000000000..0eee05a15 --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_federation.py @@ -0,0 +1,140 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, List, Literal, Optional + +import pydantic + + +class PartySpec(pydantic.BaseModel): + role: Literal["guest", "host", "arbiter", "local"] + partyid: str + + def tuple(self): + return (self.role, self.partyid) + + +class FederationPartiesSpec(pydantic.BaseModel): + local: PartySpec + parties: List[PartySpec] + + +class StandaloneFederationSpec(pydantic.BaseModel): + class MetadataSpec(pydantic.BaseModel): + federation_id: str + parties: FederationPartiesSpec + + type: Literal["standalone"] + metadata: MetadataSpec + + +class RollSiteFederationSpec(pydantic.BaseModel): + class MetadataSpec(pydantic.BaseModel): + class RollSiteConfig(pydantic.BaseModel): + host: str + port: int + + federation_id: str + parties: FederationPartiesSpec + rollsite_config: RollSiteConfig + + type: Literal["rollsite"] + metadata: MetadataSpec + + +class RabbitMQFederationSpec(pydantic.BaseModel): + class MetadataSpec(pydantic.BaseModel): + class RouteTable(pydantic.BaseModel): + host: str + port: int + + class RabbitMQConfig(pydantic.BaseModel): + host: str + port: int + mng_port: int + user: str + password: str + max_message_size: Optional[int] = None + mode: str = "replication" + + federation_id: str + parties: FederationPartiesSpec + route_table: Dict[str, RouteTable] + rabbitmq_config: RabbitMQConfig + rabbitmq_run: dict = {} + connection: dict = {} + + type: Literal["rabbitmq"] + metadata: MetadataSpec + + +class PulsarFederationSpec(pydantic.BaseModel): + class MetadataSpec(pydantic.BaseModel): + class RouteTable(pydantic.BaseModel): + class Route(pydantic.BaseModel): + host: str + port: int + sslPort: int + proxy: str = "" + + class Default(pydantic.BaseModel): + domain: str + brokerPort: int + brokerSslPort: int + proxy: str = "" + + route: Dict[str, Route] + default: Optional[Default] = None + + class PulsarConfig(pydantic.BaseModel): + host: str + port: int + mng_port: int + user: Optional[str] = None + password: Optional[str] = None + max_message_size: Optional[int] = None + mode: str = "replication" + topic_ttl: Optional[int] = None + cluster: Optional[str] = None + tenant: Optional[str] = None + + federation_id: str + parties: FederationPartiesSpec + route_table: RouteTable + pulsar_config: PulsarConfig + pulsar_run: dict = {} + connection: dict = {} + + type: Literal["pulsar"] + metadata: MetadataSpec + + +class OSXFederationSpec(pydantic.BaseModel): + class MetadataSpec(pydantic.BaseModel): + class OSXConfig(pydantic.BaseModel): + host: str + port: int + max_message_size: Optional[int] = None + + federation_id: str + parties: FederationPartiesSpec + osx_config: OSXConfig + + type: Literal["osx"] + metadata: MetadataSpec + + +class CustomFederationSpec(pydantic.BaseModel): + type: Literal["custom"] + metadata: dict diff --git a/python/fate_flow/entity/spec/dag/_job.py b/python/fate_flow/entity/spec/dag/_job.py new file mode 100644 index 000000000..820c49cf5 --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_job.py @@ -0,0 +1,94 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Optional, Union, Literal, Dict, List, Any, Tuple + +from pydantic import BaseModel + +from fate_flow.entity.spec.dag._output import OutputArtifacts +from fate_flow.entity.spec.dag._party import PartySpec +from fate_flow.entity.spec.dag._artifact import RuntimeInputArtifacts + + +class TaskSpec(BaseModel): + component_ref: str + dependent_tasks: Optional[List[str]] + parameters: Optional[Dict[Any, Any]] + inputs: Optional[RuntimeInputArtifacts] + outputs: Optional[OutputArtifacts] + parties: Optional[List[PartySpec]] + conf: Optional[Dict[Any, Any]] + stage: Optional[Union[Literal["train", "predict", "default", "cross_validation"]]] + + +class PartyTaskRefSpec(BaseModel): + parameters: Optional[Dict[Any, Any]] + conf: Optional[Dict] + + +class PartyTaskSpec(BaseModel): + parties: Optional[List[PartySpec]] + tasks: Optional[Dict[str, PartyTaskRefSpec]] = {} + conf: Optional[dict] = {} + + +class EngineRunSpec(BaseModel): + name: str + conf: Optional[Dict] + + +class TaskConfSpec(BaseModel): + engine_run: Optional[Dict] + provider: Optional[str] + timeout: Optional[int] + launcher_name: Optional[str] = "default" + + +class InheritConfSpec(BaseModel): + job_id: str + task_list: List[str] + + +class JobConfSpec(BaseModel): + class PipelineModel(BaseModel): + model_id: str + model_version: Union[str, int] + priority: Optional[int] + scheduler_party_id: Optional[str] + initiator_party_id: Optional[str] + inheritance: Optional[InheritConfSpec] + cores: Optional[int] + computing_partitions: Optional[int] + sync_type: Optional[Union[Literal["poll", "callback"]]] + auto_retries: Optional[int] + model_id: Optional[str] + model_version: Optional[Union[str, int]] + model_warehouse: Optional[PipelineModel] + task: Optional[TaskConfSpec] + extra: Optional[Dict[Any, Any]] + + +class DAGSpec(BaseModel): + parties: List[PartySpec] + conf: Optional[JobConfSpec] + stage: Optional[Union[Literal["train", "predict", "default", "cross_validation"]]] + tasks: Dict[str, TaskSpec] + party_tasks: Optional[Dict[str, PartyTaskSpec]] + + +class DAGSchema(BaseModel): + dag: Union[DAGSpec, Any] + schema_version: str + kind: str = "fate" diff --git a/python/fate_flow/entity/spec/dag/_logger.py b/python/fate_flow/entity/spec/dag/_logger.py new file mode 100644 index 000000000..677812af3 --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_logger.py @@ -0,0 +1,177 @@ +import logging +import logging.config +import os + +import pydantic +from typing import Optional + +_LOGGER_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR"] + + +class FlowLogger(pydantic.BaseModel): + config: dict + + def install(self): + for _name, _conf in self.config.get("handlers", {}).items(): + if _conf.get("filename"): + os.makedirs(os.path.dirname(_conf.get("filename")), exist_ok=True) + logging.config.dictConfig(self.config) + + @classmethod + def create( + cls, + task_log_dir: str, + job_party_log_dir: Optional[str], + level: str, + delay: bool, + formatters: Optional[dict] = None, + ): + return FlowLogger( + config=LoggerConfigBuilder( + level, formatters, delay, task_log_dir, job_party_log_dir + ).build() + ) + + +class LoggerConfigBuilder: + def __init__(self, level, formatters, delay, log_base_dir, aggregate_log_base_dir): + self.version = 1 + self.formatters = formatters + if self.formatters is None: + default_format = ( + "[%(levelname)s][%(asctime)-8s][%(process)s][%(module)s.%(funcName)s][line:%(lineno)d]: %(message)s" + ) + self.formatters = { + "root": {"format": default_format}, + "component": {"format": default_format}, + } + self.handlers = {} + self.filters = {} + self.loggers = {} + self.root = { + "handlers": [], + "level": level, + } + self.disable_existing_loggers = False + + # add loggers + root_logger_dir = os.path.join(log_base_dir, "root") + # os.makedirs(root_logger_dir, exist_ok=True) + self._add_root_loggers( + log_base_dir=root_logger_dir, formatter_name="root", delay=delay, loglevel=level + ) + + component_logger_dir = os.path.join(log_base_dir, "component") + # os.makedirs(component_logger_dir, exist_ok=True) + self._add_component_loggers( + log_base_dir=component_logger_dir, + formatter_name="component", + delay=delay, + loglevel=level, + ) + + # os.makedirs(aggregate_log_base_dir, exist_ok=True) + self._add_party_id_loggers( + aggregate_log_base_dir=aggregate_log_base_dir, formatter_name="root", delay=delay, loglevel=level + ) + + if aggregate_log_base_dir is not None: + self._add_aggregate_error_logger( + aggregate_log_base_dir, formatter_name="root", delay=delay, + ) + + def build(self): + return dict( + version=self.version, + formatters=self.formatters, + handlers=self.handlers, + filters=self.filters, + loggers=self.loggers, + root=self.root, + disable_existing_loggers=self.disable_existing_loggers, + ) + + def _add_root_loggers(self, log_base_dir, formatter_name, delay, loglevel): + for level in _LOGGER_LEVELS: + if logging.getLevelName(level) < logging.getLevelName(loglevel): + continue + handler_name = f"root_{level.lower()}" + self.handlers[handler_name] = self._create_file_handler( + level, formatter_name, delay, os.path.join(log_base_dir, level) + ) + self.root["handlers"].append(handler_name) + + def _add_party_id_loggers(self, aggregate_log_base_dir, formatter_name, delay, loglevel): + for level in _LOGGER_LEVELS: + if logging.getLevelName(level) < logging.getLevelName(loglevel): + continue + handler_name = f"root_party_{level.lower()}" + self.handlers[handler_name] = self._create_file_handler( + level, formatter_name, delay, os.path.join(aggregate_log_base_dir, level) + ) + self.root["handlers"].append(handler_name) + + def _add_aggregate_error_logger(self, log_base_dir, formatter_name, delay): + # error from all component + handler_name = "global_error" + self.handlers[handler_name] = self._create_file_handler( + "ERROR", formatter_name, delay, os.path.join(log_base_dir, "ERROR") + ) + self.root["handlers"].append(handler_name) + + def _add_component_loggers( + self, log_base_dir, formatter_name: str, loglevel: str, delay: bool + ): + # basic component logger handlers + # logger structure: + # component/ + # DEBUG + # INFO + # WARNING + # ERROR + component_handlers_names = [] + for level in _LOGGER_LEVELS: + if logging.getLevelName(level) < logging.getLevelName(loglevel): + continue + handler_name = f"component_{level.lower()}" + self.handlers[handler_name] = self._create_file_handler( + level, formatter_name, delay, os.path.join(log_base_dir, level) + ) + component_handlers_names.append(handler_name) + + # add profile logger handler + # logger structure: + # component/ + # PROFILE + handler_name = "component_profile" + filter_name = "component_profile_filter" + self.filters[filter_name] = { + "name": "fate.arch.computing._profile", + "()": "logging.Filter", + } + self.handlers[handler_name] = self._create_file_handler( + "DEBUG", + formatter_name, + delay, + os.path.join(log_base_dir, "PROFILE"), + [filter_name], + ) + component_handlers_names.append(handler_name) + + # the "fate" name means the logger only log the logs from fate package + # so, don't change the name or the logger will not work + self.loggers["fate"] = dict( + handlers=component_handlers_names, + level=loglevel, + ) + + @staticmethod + def _create_file_handler(level, formatter, delay, filename, filters=None): + return { + "class": "logging.FileHandler", + "level": level, + "formatter": formatter, + "delay": delay, + "filename": filename, + "filters": [] if filters is None else filters, + } diff --git a/python/fate_flow/entity/spec/dag/_mlmd.py b/python/fate_flow/entity/spec/dag/_mlmd.py new file mode 100644 index 000000000..4fa98e83d --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_mlmd.py @@ -0,0 +1,29 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Optional, Any, Dict, Union + +import pydantic + + +class FlowMLMDMetadata(pydantic.BaseModel): + host: Optional[str] + port: Optional[int] + protocol: Optional[str] + + +class MLMDSpec(pydantic.BaseModel): + type: str + metadata: Union[Dict[str, Any], FlowMLMDMetadata] diff --git a/python/fate_flow/entity/spec/dag/_output.py b/python/fate_flow/entity/spec/dag/_output.py new file mode 100644 index 000000000..b9bdcfc6a --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_output.py @@ -0,0 +1,128 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Literal, Union, List, Dict, Optional + +import pydantic +from pydantic import typing + +from ._party import PartySpec + + +class MetricData(pydantic.BaseModel): + class Group(pydantic.BaseModel): + name: str + index: Optional[int] + name: str + type: Optional[str] + groups: List[Group] = [] + step_axis: Optional[str] + data: Union[List, Dict] + + +class DirectoryDataPool(pydantic.BaseModel): + class DirectoryDataPoolMetadata(pydantic.BaseModel): + uri: str + format: str = "csv" + name_template: str = "{name}" # `name` and `uuid` allowed in template + + type: Literal["directory"] + metadata: DirectoryDataPoolMetadata + + +class CustomDataPool(pydantic.BaseModel): + type: Literal["custom"] + metadata: dict + + +class DirectoryModelPool(pydantic.BaseModel): + class DirectoryDataPoolMetadata(pydantic.BaseModel): + uri: str + format: str = "json" + name_template: str = "{name}" # `name` and `uuid` allowed in template + + type: Literal["directory"] + metadata: DirectoryDataPoolMetadata + + +class CustomModelPool(pydantic.BaseModel): + type: Literal["custom"] + metadata: dict + + +class DirectoryMetricPool(pydantic.BaseModel): + class DirectoryDataPoolMetadata(pydantic.BaseModel): + uri: str + format: str = "json" + name_template: str = "{name}" # `name` and `uuid` allowed in template + + type: Literal["directory"] + metadata: DirectoryDataPoolMetadata + + +class CustomMetricPool(pydantic.BaseModel): + type: Literal["custom"] + metadata: dict + + +class OutputPoolConf(pydantic.BaseModel): + data: Union[DirectoryDataPool, CustomDataPool] + model: Union[DirectoryModelPool, CustomModelPool] + metric: Union[DirectoryMetricPool, CustomMetricPool] + + +class IOMeta(pydantic.BaseModel): + class InputMeta(pydantic.BaseModel): + data: typing.Dict[str, Union[List[Dict], Dict]] + model: typing.Dict[str, Union[List[Dict], Dict]] + + class OutputMeta(pydantic.BaseModel): + data: typing.Dict[str, Union[List[Dict], Dict]] + model: typing.Dict[str, Union[List[Dict], Dict]] + metric: typing.Dict[str, Union[List[Dict], Dict]] + + inputs: InputMeta + outputs: OutputMeta + + +class ComponentOutputMeta(pydantic.BaseModel): + class Status(pydantic.BaseModel): + code: int + exceptions: typing.Optional[str] + status: Status + io_meta: typing.Optional[IOMeta] + + +class OutputArtifactSpec(pydantic.BaseModel): + output_artifact_key_alias: str + output_artifact_type_alias: str + parties: Optional[List[PartySpec]] + + +class OutputArtifacts(pydantic.BaseModel): + data: Optional[Dict[str, Union[OutputArtifactSpec, List[OutputArtifactSpec]]]] + model: Optional[Dict[str, Union[OutputArtifactSpec, List[OutputArtifactSpec]]]] + metric: Optional[Dict[str, Union[OutputArtifactSpec, List[OutputArtifactSpec]]]] + + +class OutputArtifactType(object): + DATA = "data" + MODEL = "model" + METRIC = "metric" + + @classmethod + def types(cls): + for _type in [cls.DATA, cls.MODEL, cls.METRIC]: + yield _type + diff --git a/python/fate_flow/entity/spec/dag/_party.py b/python/fate_flow/entity/spec/dag/_party.py new file mode 100644 index 000000000..cb8d9331e --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_party.py @@ -0,0 +1,25 @@ +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import Union, Literal, List + +from pydantic import BaseModel + + +class PartySpec(BaseModel): + role: Union[Literal["guest", "host", "arbiter", "local"]] + party_id: List[str] + + def tuple(self): + return self.role, self.party_id diff --git a/python/fate_flow/entity/spec/dag/_task.py b/python/fate_flow/entity/spec/dag/_task.py new file mode 100644 index 000000000..7b6a78072 --- /dev/null +++ b/python/fate_flow/entity/spec/dag/_task.py @@ -0,0 +1,74 @@ +from typing import Optional, Union, Dict, Any, List + +import pydantic + +from fate_flow.entity.spec.dag._artifact import ArtifactInputApplySpec, ArtifactOutputApplySpec, \ + FlowRuntimeInputArtifacts +from fate_flow.entity.spec.dag._computing import StandaloneComputingSpec, SparkComputingSpec, EggrollComputingSpec +from fate_flow.entity.spec.dag._device import CPUSpec, GPUSpec +from fate_flow.entity.spec.dag._federation import StandaloneFederationSpec, RollSiteFederationSpec, RabbitMQFederationSpec,PulsarFederationSpec,OSXFederationSpec +from fate_flow.entity.spec.dag._logger import FlowLogger +from fate_flow.entity.spec.dag._mlmd import MLMDSpec + + +class TaskRuntimeConfSpec(pydantic.BaseModel): + device: Union[CPUSpec, GPUSpec] + computing: Union[StandaloneComputingSpec, EggrollComputingSpec, SparkComputingSpec] + storage: Optional[str] + federation: Union[ + StandaloneFederationSpec, + RollSiteFederationSpec, + RabbitMQFederationSpec, + PulsarFederationSpec, + OSXFederationSpec, + ] + logger: Union[FlowLogger] + + +class PreTaskConfigSpec(pydantic.BaseModel): + model_id: Optional[str] = "" + model_version: Optional[str] = "" + job_id: Optional[str] = "" + task_id: str + task_version: str + task_name: str + provider_name: str = "fate" + party_task_id: str + component: str + role: str + party_id: str + stage: str = "default" + parameters: Dict[str, Any] = {} + input_artifacts: FlowRuntimeInputArtifacts = {} + conf: TaskRuntimeConfSpec + mlmd: MLMDSpec + engine_run: Optional[Dict[str, Any]] = {} + computing_partitions: int = None + launcher_name: Optional[str] = "default" + launcher_conf: Optional[Dict] = {} + + +class TaskConfigSpec(pydantic.BaseModel): + job_id: Optional[str] = "" + task_id: str + party_task_id: str + task_name: str + component: str + role: str + party_id: str + stage: str = "default" + parameters: Dict[str, Any] = {} + input_artifacts: Optional[Dict[str, Union[List[ArtifactInputApplySpec], ArtifactInputApplySpec, None]]] = {} + output_artifacts: Optional[Dict[str, Union[ArtifactOutputApplySpec, None]]] = {} + conf: TaskRuntimeConfSpec + + +class TaskCleanupConfigSpec(pydantic.BaseModel): + computing: Union[StandaloneComputingSpec, EggrollComputingSpec, SparkComputingSpec] + federation: Union[ + StandaloneFederationSpec, + RollSiteFederationSpec, + RabbitMQFederationSpec, + PulsarFederationSpec, + OSXFederationSpec, + ] diff --git a/python/fate_flow/entity/spec/flow/__init__.py b/python/fate_flow/entity/spec/flow/__init__.py new file mode 100644 index 000000000..73ba8250a --- /dev/null +++ b/python/fate_flow/entity/spec/flow/__init__.py @@ -0,0 +1,25 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ._model import MLModelSpec, Metadata +from ._storage import FileStorageSpec, MysqlStorageSpec, TencentCosStorageSpec +from ._provider import ProviderSpec, DockerProviderSpec, K8sProviderSpec, LocalProviderSpec +from ._scheduler import SchedulerInfoSpec +from ._protocol import SubmitJobInput, SubmitJobOutput, QueryJobInput, QueryJobOutput, StopJobInput, StopJobOutput, \ + QueryTaskOutput, QueryTaskInput + +__all__ = ["MLModelSpec", "FileStorageSpec", "MysqlStorageSpec", "TencentCosStorageSpec", "ProviderSpec", + "DockerProviderSpec", "K8sProviderSpec", "LocalProviderSpec", "SchedulerInfoSpec", "Metadata", + "SubmitJobInput", "SubmitJobOutput", "QueryJobInput", "QueryJobOutput", "StopJobInput", "StopJobOutput", + "QueryTaskInput", "QueryTaskOutput"] diff --git a/python/fate_flow/entity/spec/flow/_model.py b/python/fate_flow/entity/spec/flow/_model.py new file mode 100644 index 000000000..7ae73f07c --- /dev/null +++ b/python/fate_flow/entity/spec/flow/_model.py @@ -0,0 +1,67 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from datetime import datetime +from typing import List, Optional, Dict, Any, Union + +import pydantic + + +class MLModelComponentSpec(pydantic.BaseModel): + name: str + provider: str + version: str + metadata: dict + + +class MLModelPartiesSpec(pydantic.BaseModel): + guest: List[str] + host: List[str] + arbiter: List[str] + + +class MLModelFederatedSpec(pydantic.BaseModel): + + task_id: str + parties: MLModelPartiesSpec + component: MLModelComponentSpec + + +class MLModelModelSpec(pydantic.BaseModel): + name: str + created_time: datetime + file_format: str + metadata: dict + + +class MLModelPartySpec(pydantic.BaseModel): + + party_task_id: str + role: str + partyid: str + models: List[MLModelModelSpec] + + +class MLModelSpec(pydantic.BaseModel): + + federated: MLModelFederatedSpec + party: MLModelPartySpec + + +class Metadata(pydantic.BaseModel): + metadata: dict + model_overview: MLModelSpec + model_key: str + index: Optional[Union[int, None]] = None + source: Optional[Dict[str, Any]] = None diff --git a/python/fate_flow/entity/spec/flow/_protocol.py b/python/fate_flow/entity/spec/flow/_protocol.py new file mode 100644 index 000000000..a430953da --- /dev/null +++ b/python/fate_flow/entity/spec/flow/_protocol.py @@ -0,0 +1,41 @@ +from typing import Optional, Dict, Any, List + +import pydantic + +from fate_flow.entity.spec.dag import DAGSchema + + +class SubmitJobInput(pydantic.BaseModel): + dag_schema: DAGSchema + + +class SubmitJobOutput(pydantic.BaseModel): + message: str = "success" + code: int = 0 + job_id: str + data: Optional[Dict[str, Any]] = {} + + +class QueryJobInput(pydantic.BaseModel): + jobs: List[Any] + + +class QueryJobOutput(pydantic.BaseModel): + jobs: List[Any] + + +class StopJobInput(pydantic.BaseModel): + job_id: str + + +class StopJobOutput(pydantic.BaseModel): + message: str = "success" + code: int = 0 + + +class QueryTaskInput(pydantic.BaseModel): + tasks: List[Any] + + +class QueryTaskOutput(pydantic.BaseModel): + tasks: List[Any] diff --git a/python/fate_flow/entity/spec/flow/_provider.py b/python/fate_flow/entity/spec/flow/_provider.py new file mode 100644 index 000000000..a61758c83 --- /dev/null +++ b/python/fate_flow/entity/spec/flow/_provider.py @@ -0,0 +1,59 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from typing import Optional, Union + +import pydantic + +from fate_flow.errors.server_error import FileNoFound + + +class BaseProvider(pydantic.BaseModel): + def __init__(self, check=False, **kwargs): + super(BaseProvider, self).__init__(**kwargs) + if check: + self.check() + + def check(self): + pass + + +class LocalProviderSpec(BaseProvider): + def check(self): + if not os.path.exists(self.path): + raise FileNoFound(path=self.path) + if self.venv and not os.path.exists(self.venv): + raise FileNoFound(venv=self.venv) + + path: str + venv: Optional[str] + + +class DockerProviderSpec(BaseProvider): + base_url: str + image: str + + +class K8sProviderSpec(BaseProvider): + image: str + namespace: str + config: Optional[dict] + + +class ProviderSpec(BaseProvider): + name: str + version: str + device: str + metadata: Union[LocalProviderSpec, DockerProviderSpec, K8sProviderSpec] diff --git a/python/fate_flow/entity/spec/flow/_scheduler.py b/python/fate_flow/entity/spec/flow/_scheduler.py new file mode 100644 index 000000000..2acf82f17 --- /dev/null +++ b/python/fate_flow/entity/spec/flow/_scheduler.py @@ -0,0 +1,31 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, List, Union, Dict + +from pydantic import BaseModel + +from fate_flow.entity.spec.dag import PartySpec + + +class SchedulerInfoSpec(BaseModel): + dag: Dict[str, Any] + parties: List[PartySpec] + initiator_party_id: str + scheduler_party_id: str + federated_status_collect_type: str + model_id: str + model_version: Union[str, int] + + diff --git a/python/fate_flow/entity/spec/flow/_storage.py b/python/fate_flow/entity/spec/flow/_storage.py new file mode 100644 index 000000000..3177f458c --- /dev/null +++ b/python/fate_flow/entity/spec/flow/_storage.py @@ -0,0 +1,38 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Union + +import pydantic + + +class FileStorageSpec(pydantic.BaseModel): + path: Union[str, None] + + +class MysqlStorageSpec(pydantic.BaseModel): + name: str + user: str + passwd: str + host: str + port: int + max_connections: int + stale_timeout: int + + +class TencentCosStorageSpec(pydantic.BaseModel): + Region: str + SecretId: str + SecretKey: str + Bucket: str diff --git a/python/fate_flow/entity/tests/job_test.py b/python/fate_flow/entity/tests/job_test.py deleted file mode 100644 index d61ec2955..000000000 --- a/python/fate_flow/entity/tests/job_test.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import unittest -from fate_flow.entity import JobConfigurationBase, JobConfiguration - - -class TestJobConfiguration(unittest.TestCase): - def test(self): - self.assertEqual(JobConfigurationBase(**{"runtime_conf": {"conf": 0}, "dsl": {"dsl"}}).runtime_conf.get("conf"), 0) - self.assertEqual(JobConfigurationBase(**{"job_runtime_conf": {"conf": 0}, "job_dsl": {"dsl"}}).runtime_conf.get("conf"), 0) - self.assertEqual(JobConfiguration(**{"runtime_conf": {"conf": 0}, "dsl": {"dsl"}, "runtime_conf_on_party": {"conf": 2}, "train_runtime_conf": {"conf": 2}}).runtime_conf.get("conf"), 0) - self.assertEqual(JobConfiguration(**{"runtime_conf": {"conf": 0}, "dsl": {"dsl"}, "runtime_conf_on_party": {"conf": 2}, "train_runtime_conf": {"conf": 2}}).runtime_conf_on_party.get("conf"), 2) - - -if __name__ == '__main__': - unittest.main() diff --git a/python/fate_flow/entity/tests/type_test.py b/python/fate_flow/entity/tests/type_test.py deleted file mode 100644 index 309864b78..000000000 --- a/python/fate_flow/entity/tests/type_test.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import unittest -from fate_arch.common.base_utils import json_dumps, json_loads -from fate_flow.utils.object_utils import from_dict_hook -from fate_flow.entity.types import * -from fate_flow.entity import DataCache -from fate_arch.common import DTable - - -class TestType(unittest.TestCase): - def test1(self): - cache = DataCache(name="test_cache", data={"t1": DTable(namespace="test", name="test1")}, meta={"t1": {"a": 1}}) - a = json_loads(json_dumps(cache)) - self.assertEqual(a["data"]["t1"]["namespace"], "test") - b = json_loads(json_dumps(cache, with_type=True), object_hook=from_dict_hook) - self.assertEqual(b.data["t1"].namespace, "test") - - def test2(self): - self.assertTrue(ComponentProviderName.valid("fate_flow")) - self.assertFalse(ComponentProviderName.valid("fate_flow_xx")) - self.assertTrue(ComponentProviderName("fate_flow") in ComponentProviderName) - self.assertTrue(ComponentProviderName["FATE_FLOW"] in ComponentProviderName) - self.assertTrue(ComponentProviderName("fate_flow") == ComponentProviderName.FATE_FLOW) - self.assertTrue(ComponentProviderName("fate_flow") is ComponentProviderName.FATE_FLOW) - print(ComponentProviderName.values()) - print(ComponentProviderName.FATE_FLOW) - - self.assertTrue(0 == KillProcessRetCode.KILLED) - self.assertTrue(KillProcessRetCode.valid(0)) - self.assertFalse(KillProcessRetCode.valid(10)) - print(KillProcessRetCode.KILLED) - - -if __name__ == '__main__': - unittest.main() diff --git a/python/fate_flow/entity/types.py b/python/fate_flow/entity/types.py deleted file mode 100644 index 7797062c5..000000000 --- a/python/fate_flow/entity/types.py +++ /dev/null @@ -1,155 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from enum import IntEnum, Enum - - -class CustomEnum(Enum): - @classmethod - def valid(cls, value): - try: - cls(value) - return True - except: - return False - - @classmethod - def values(cls): - return [member.value for member in cls.__members__.values()] - - @classmethod - def names(cls): - return [member.name for member in cls.__members__.values()] - - -class ComponentProviderName(CustomEnum): - FATE = "fate" - FATE_FLOW = "fate_flow" - FATE_SQL = "fate_sql" - - -class FateDependenceName(CustomEnum): - Fate_Source_Code = "fate_code" - Python_Env = "python_env" - - -class FateDependenceStorageEngine(CustomEnum): - HDFS = "HDFS" - - -class PythonDependenceName(CustomEnum): - Fate_Source_Code = "python" - Python_Env = "miniconda" - - -class ModelStorage(CustomEnum): - REDIS = "redis" - MYSQL = "mysql" - TENCENT_COS = "tencent_cos" - - -class ModelOperation(CustomEnum): - STORE = "store" - RESTORE = "restore" - EXPORT = "export" - IMPORT = "import" - LOAD = "load" - BIND = "bind" - - -class ProcessRole(CustomEnum): - DRIVER = "driver" - WORKER = "worker" - - -class TagOperation(CustomEnum): - CREATE = "create" - RETRIEVE = "retrieve" - UPDATE = "update" - DESTROY = "destroy" - LIST = "list" - - -class ResourceOperation(CustomEnum): - APPLY = "apply" - RETURN = "return" - - -class PermissionType(CustomEnum): - COMPONENT = "component" - DATASET = "dataset" - - -class SiteKeyName(CustomEnum): - PRIVATE = "private" - PUBLIC = "public" - - -class RegistryServiceName(CustomEnum): - UPLOAD = "upload" - DOWNLOAD = "download" - QUERY = "query" - CLIENT_AUTHENTICATION = "client_authentication" - SIGNATURE = "signature" - SITE_AUTHENTICATION = "site_authentication" - PERMISSION_CHECK = "permission" - - -class KillProcessRetCode(IntEnum, CustomEnum): - KILLED = 0 - NOT_FOUND = 1 - ERROR_PID = 2 - - -class InputSearchType(IntEnum, CustomEnum): - UNKNOWN = 0 - TABLE_INFO = 1 - JOB_COMPONENT_OUTPUT = 2 - - -class RetCode(IntEnum, CustomEnum): - SUCCESS = 0 - NOT_EFFECTIVE = 10 - EXCEPTION_ERROR = 100 - ARGUMENT_ERROR = 101 - DATA_ERROR = 102 - OPERATING_ERROR = 103 - FEDERATED_ERROR = 104 - CONNECTION_ERROR = 105 - RUNNING = 106 - INCOMPATIBLE_FATE_VER = 107 - PERMISSION_ERROR = 108 - AUTHENTICATION_ERROR = 109 - SERVER_ERROR = 500 - - -class WorkerName(CustomEnum): - TASK_EXECUTOR = "task_executor" - TASK_INITIALIZER = "task_initializer" - PROVIDER_REGISTRAR = "provider_registrar" - DEPENDENCE_UPLOAD = "dependence_upload" - -class TaskCleanResourceType(CustomEnum): - TABLE = "table" - METRICS = "metrics" - - -class ExternalStorage(CustomEnum): - MYSQL = "MYSQL" - - -class TaskLauncher(CustomEnum): - DEFAULT = "default" - DEEPSPEED = "deepspeed" diff --git a/python/fate_flow/entity/types/__init__.py b/python/fate_flow/entity/types/__init__.py new file mode 100644 index 000000000..faa264a65 --- /dev/null +++ b/python/fate_flow/entity/types/__init__.py @@ -0,0 +1,28 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from ._work import * +from ._address import * +from ._engine import * +from ._output import * +from ._provider import * +from ._status import * +from ._federation import * +from ._command import * +from ._api import * +from ._instance import * +from ._permission import * +from ._input import * +from ._artificats import * diff --git a/python/fate_flow/entity/types/_address.py b/python/fate_flow/entity/types/_address.py new file mode 100644 index 000000000..62d1768fa --- /dev/null +++ b/python/fate_flow/entity/types/_address.py @@ -0,0 +1,237 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import abc + + +class AddressABC(metaclass=abc.ABCMeta): + ... + + +class AddressBase(AddressABC): + def __init__(self, connector_name=None): + pass + + @property + def connector(self): + return {} + + @property + def storage_engine(self): + return + + @property + def engine_path(self): + return + + +class StandaloneAddress(AddressBase): + def __init__(self, home=None, name=None, namespace=None, storage_type=None, connector_name=None): + self.home = home + self.name = name + self.namespace = namespace + self.storage_type = storage_type + super(StandaloneAddress, self).__init__(connector_name=connector_name) + + def __hash__(self): + return (self.home, self.name, self.namespace, self.storage_type).__hash__() + + def __str__(self): + return f"StandaloneAddress(name={self.name}, namespace={self.namespace})" + + def __repr__(self): + return self.__str__() + + @property + def connector(self): + return {"home": self.home} + + @property + def engine_path(self): + if self.home: + return f"standalone:///{self.home}/{self.namespace}/{self.name}" + else: + return f"standalone:///{self.namespace}/{self.name}" + + +class EggRollAddress(AddressBase): + def __init__(self, home=None, name=None, namespace=None, connector_name=None): + self.name = name + self.namespace = namespace + self.home = home + super(EggRollAddress, self).__init__(connector_name=connector_name) + + def __hash__(self): + return (self.home, self.name, self.namespace).__hash__() + + def __str__(self): + return f"EggRollAddress(name={self.name}, namespace={self.namespace})" + + def __repr__(self): + return self.__str__() + + @property + def connector(self): + return {"home": self.home} + + @property + def engine_path(self): + return f"eggroll:///{self.namespace}/{self.name}" + + +class HDFSAddress(AddressBase): + def __init__(self, name_node=None, path=None, connector_name=None): + self.name_node = name_node + self.path = path + super(HDFSAddress, self).__init__(connector_name=connector_name) + + def __hash__(self): + return (self.name_node, self.path).__hash__() + + def __str__(self): + return f"HDFSAddress(name_node={self.name_node}, path={self.path})" + + def __repr__(self): + return self.__str__() + + @property + def engine_path(self): + if not self.name_node: + return f"hdfs://{self.path}" + else: + if "hdfs" not in self.name_node: + return f"hdfs://{self.name_node}{self.path}" + else: + return f"{self.name_node}{self.path}" + + @property + def connector(self): + return {"name_node": self.name_node} + + +class PathAddress(AddressBase): + def __init__(self, path=None, connector_name=None): + self.path = path + super(PathAddress, self).__init__(connector_name=connector_name) + + def __hash__(self): + return self.path.__hash__() + + def __str__(self): + return f"PathAddress(path={self.path})" + + def __repr__(self): + return self.__str__() + + @property + def engine_path(self): + return f"file://{self.path}" + + +class ApiAddress(AddressBase): + def __init__(self, method="POST", url=None, header=None, body=None, connector_name=None): + self.method = method + self.url = url + self.header = header if header else {} + self.body = body if body else {} + super(ApiAddress, self).__init__(connector_name=connector_name) + + def __hash__(self): + return (self.method, self.url).__hash__() + + def __str__(self): + return f"ApiAddress(url={self.url})" + + def __repr__(self): + return self.__str__() + + @property + def engine_path(self): + return self.url + + +class MysqlAddress(AddressBase): + def __init__(self, user=None, passwd=None, host=None, port=None, db=None, name=None, connector_name=None): + self.user = user + self.passwd = passwd + self.host = host + self.port = port + self.db = db + self.name = name + self.connector_name = connector_name + super(MysqlAddress, self).__init__(connector_name=connector_name) + + def __hash__(self): + return (self.host, self.port, self.db, self.name).__hash__() + + def __str__(self): + return f"MysqlAddress(db={self.db}, name={self.name})" + + def __repr__(self): + return self.__str__() + + @property + def connector(self): + return {"user": self.user, "passwd": self.passwd, "host": self.host, "port": self.port, "db": self.db} + + +class HiveAddress(AddressBase): + def __init__(self, host=None, name=None, port=10000, username=None, database='default', auth_mechanism='PLAIN', + password=None, connector_name=None): + self.host = host + self.username = username + self.port = port + self.database = database + self.auth_mechanism = auth_mechanism + self.password = password + self.name = name + super(HiveAddress, self).__init__(connector_name=connector_name) + + def __hash__(self): + return (self.host, self.port, self.database, self.name).__hash__() + + def __str__(self): + return f"HiveAddress(database={self.database}, name={self.name})" + + def __repr__(self): + return self.__str__() + + @property + def connector(self): + return { + "host": self.host, + "port": self.port, + "username": self.username, + "password": self.password, + "auth_mechanism": self.auth_mechanism, + "database": self.database} + + +class FileAddress(AddressBase): + def __init__(self, path=None, connector_name=None): + self.path = path + super(FileAddress, self).__init__(connector_name=connector_name) + + def __hash__(self): + return self.path.__hash__() + + def __str__(self): + return f"FileAddress(path={self.path})" + + def __repr__(self): + return self.__str__() + + @property + def engine_path(self): + return f"file://{self.path}" diff --git a/python/fate_flow/entity/types/_api.py b/python/fate_flow/entity/types/_api.py new file mode 100644 index 000000000..7604df18f --- /dev/null +++ b/python/fate_flow/entity/types/_api.py @@ -0,0 +1,23 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +class AppType: + SITE = "site" + CLIENT = "client" + + +class PROTOCOL: + FATE_FLOW = "fate" + BFIA = "bfia" diff --git a/python/fate_flow/entity/types/_artificats.py b/python/fate_flow/entity/types/_artificats.py new file mode 100644 index 000000000..fdd5520cf --- /dev/null +++ b/python/fate_flow/entity/types/_artificats.py @@ -0,0 +1,60 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from typing import List + +from typing_extensions import Protocol + + +class ArtifactType(Protocol): + type_name: str + path_type: str + uri_types: List[str] + + +class DataframeArtifactType(ArtifactType): + type_name = "dataframe" + path_type = "distributed" + uri_types = ["eggroll", "hdfs"] + + +class TableArtifactType(ArtifactType): + type_name = "table" + path_type = "distributed" + uri_types = ["eggroll", "hdfs"] + + +class DataDirectoryArtifactType(ArtifactType): + type_name = "data_directory" + path_type = "directory" + uri_types = ["file"] + + +class ModelDirectoryArtifactType(ArtifactType): + type_name = "model_directory" + path_type = "directory" + uri_types = ["file"] + + +class JsonModelArtifactType(ArtifactType): + type_name = "json_model" + path_type = "file" + uri_types = ["file"] + + +class JsonMetricArtifactType(ArtifactType): + type_name = "json_metric" + path_type = "file" + uri_types = ["file"] diff --git a/python/fate_flow/entity/types/_command.py b/python/fate_flow/entity/types/_command.py new file mode 100644 index 000000000..6775a750f --- /dev/null +++ b/python/fate_flow/entity/types/_command.py @@ -0,0 +1,21 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.entity import CustomEnum + + +class ResourceOperation(CustomEnum): + APPLY = "apply" + RETURN = "return" diff --git a/python/fate_flow/entity/types/_engine.py b/python/fate_flow/entity/types/_engine.py new file mode 100644 index 000000000..ea68c6b81 --- /dev/null +++ b/python/fate_flow/entity/types/_engine.py @@ -0,0 +1,62 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +class EngineType(object): + COMPUTING = "computing" + STORAGE = "storage" + FEDERATION = "federation" + + +class FederationEngine(object): + ROLLSITE = "rollsite" + RABBITMQ = "rabbitmq" + STANDALONE = "standalone" + PULSAR = "pulsar" + OSX = "osx" + + +class ComputingEngine(object): + EGGROLL = "eggroll" + SPARK = "spark" + STANDALONE = "standalone" + + +class StorageEngine(object): + STANDALONE = "standalone" + EGGROLL = "eggroll" + HDFS = "hdfs" + MYSQL = "mysql" + SIMPLE = "simple" + PATH = "path" + HIVE = "hive" + FILE = "file" + API = "api" + + +class CoordinationProxyService(object): + ROLLSITE = "rollsite" + NGINX = "nginx" + FATEFLOW = "fateflow" + FIREWORK = "firework" + OSX = "osx" + + +class FederatedCommunicationType(object): + POLL = "poll" + CALLBACK = "callback" + + +class LauncherType(object): + DEFAULT = "default" + DEEPSPEED = "deepspeed" diff --git a/python/fate_flow/protobuf/generate_py.sh b/python/fate_flow/entity/types/_federation.py similarity index 62% rename from python/fate_flow/protobuf/generate_py.sh rename to python/fate_flow/entity/types/_federation.py index 145bd5c53..41b2e6555 100644 --- a/python/fate_flow/protobuf/generate_py.sh +++ b/python/fate_flow/entity/types/_federation.py @@ -1,5 +1,3 @@ -#!/usr/bin/env bash - # # Copyright 2019 The FATE Authors. All Rights Reserved. # @@ -15,26 +13,23 @@ # See the License for the specific language governing permissions and # limitations under the License. # +class CoordinationCommunicationProtocol(object): + HTTP = "http" + GRPC = "grpc" + + +class OSXMode(object): + STREAM = "stream" + QUEUE = "queue" -BASEDIR=$(dirname "$0") -cd "$BASEDIR" || exit -PROTO_DIR="proto" -TARGER_DIR="python" +class FederatedMode(object): + SINGLE = "SINGLE" + MULTIPLE = "MULTIPLE" -generate() { - python -m grpc_tools.protoc -I./$PROTO_DIR --python_out=./$TARGER_DIR "$1" -} + def is_single(self, value): + return value == self.SINGLE -generate_all() { - for proto in "$PROTO_DIR"/*.proto; do - echo "protoc: $proto" - generate "$proto" - done -} + def is_multiple(self, value): + return value == self.MULTIPLE -if [ $# -gt 0 ]; then - generate "$1" -else - generate_all -fi diff --git a/python/fate_flow/entity/types/_input.py b/python/fate_flow/entity/types/_input.py new file mode 100644 index 000000000..6cb21687e --- /dev/null +++ b/python/fate_flow/entity/types/_input.py @@ -0,0 +1,30 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +class ArtifactSourceType(object): + TASK_OUTPUT_ARTIFACT = "task_output_artifact" + MODEL_WAREHOUSE = "model_warehouse" + DATA_WAREHOUSE = "data_warehouse" + + +class InputArtifactType(object): + DATA = "data" + MODEL = "model" + + @classmethod + def types(cls): + for _type in [cls.DATA, cls.MODEL]: + yield _type \ No newline at end of file diff --git a/python/fate_flow/entity/instance.py b/python/fate_flow/entity/types/_instance.py similarity index 97% rename from python/fate_flow/entity/instance.py rename to python/fate_flow/entity/types/_instance.py index 94f9d338e..13c708ef4 100644 --- a/python/fate_flow/entity/instance.py +++ b/python/fate_flow/entity/types/_instance.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from ._base import BaseEntity +from .._base import BaseEntity class FlowInstance(BaseEntity): diff --git a/python/fate_flow/entity/types/_output.py b/python/fate_flow/entity/types/_output.py new file mode 100644 index 000000000..bd54038e0 --- /dev/null +++ b/python/fate_flow/entity/types/_output.py @@ -0,0 +1,27 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, List, Union, Any + +from fate_flow.entity import BaseModel + + +class ModelStorageEngine(object): + FILE = "file" + MYSQL = "mysql" + TENCENT_COS = "tencent_cos" + + +class ModelFileFormat(object): + JSON = "json" diff --git a/python/fate_flow/entity/permission_parameters.py b/python/fate_flow/entity/types/_permission.py similarity index 88% rename from python/fate_flow/entity/permission_parameters.py rename to python/fate_flow/entity/types/_permission.py index 00fbf1f29..f88a8e346 100644 --- a/python/fate_flow/entity/permission_parameters.py +++ b/python/fate_flow/entity/types/_permission.py @@ -15,10 +15,10 @@ # import json -from ._base import BaseEntity +from fate_flow.entity import CustomEnum -class PermissionParameters(BaseEntity): +class PermissionParameters(object): def __init__(self, **kwargs): self.party_id = None self.component = None @@ -37,7 +37,7 @@ def to_dict(self): return d -class DataSet(BaseEntity): +class DataSet(object): def __init__(self, namespace, name, **kwargs): self.namespace = namespace self.name = name @@ -47,7 +47,7 @@ def to_dict(self): for k, v in self.__dict__.items(): if v is None: continue - d[k] = v + d[k] = str(v) return d @property @@ -67,8 +67,6 @@ def check(self): raise ValueError(f"name {self.name} or namespace {self.namespace} is null") -class CheckReturn: - SUCCESS = 0 - NO_ROLE_PERMISSION = 1 - NO_COMPONENT_PERMISSION = 2 - NO_DATASET_PERMISSION = 3 +class PermissionType(CustomEnum): + COMPONENT = "component" + DATASET = "dataset" diff --git a/python/fate_flow/entity/metric.py b/python/fate_flow/entity/types/_provider.py similarity index 78% rename from python/fate_flow/entity/metric.py rename to python/fate_flow/entity/types/_provider.py index df2ec4eef..960158614 100644 --- a/python/fate_flow/entity/metric.py +++ b/python/fate_flow/entity/types/_provider.py @@ -12,6 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -from ._metric import Metric, MetricMeta, MetricType -# Available for use with federatedml components earlier than version 1.7 +class ProviderDevice(object): + LOCAL = "local" + DOCKER = "docker" + K8S = "k8s" + + +class ProviderName(object): + FATE = "fate" + FATE_FLOW = "fate_flow" diff --git a/python/fate_flow/entity/run_status.py b/python/fate_flow/entity/types/_status.py similarity index 88% rename from python/fate_flow/entity/run_status.py rename to python/fate_flow/entity/types/_status.py index c7c0e414c..a01812f1e 100644 --- a/python/fate_flow/entity/run_status.py +++ b/python/fate_flow/entity/types/_status.py @@ -32,6 +32,7 @@ class StatusSet(BaseStatus): FAILED = "failed" PASS = "pass" SUCCESS = "success" + FINISHED = "finished" @classmethod def get_level(cls, status): @@ -55,6 +56,7 @@ class JobStatus(BaseStatus): READY = StatusSet.READY WAITING = StatusSet.WAITING RUNNING = StatusSet.RUNNING + FINISHED = StatusSet.FINISHED CANCELED = StatusSet.CANCELED TIMEOUT = StatusSet.TIMEOUT FAILED = StatusSet.FAILED @@ -63,11 +65,11 @@ class JobStatus(BaseStatus): class StateTransitionRule(BaseStateTransitionRule): RULES = { StatusSet.READY: [StatusSet.WAITING, StatusSet.CANCELED, StatusSet.TIMEOUT, StatusSet.FAILED], - StatusSet.WAITING: [StatusSet.RUNNING, StatusSet.CANCELED, StatusSet.TIMEOUT, StatusSet.FAILED, StatusSet.SUCCESS, StatusSet.PASS], + StatusSet.WAITING: [StatusSet.RUNNING, StatusSet.CANCELED, StatusSet.TIMEOUT, StatusSet.FAILED, StatusSet.SUCCESS], StatusSet.RUNNING: [StatusSet.CANCELED, StatusSet.TIMEOUT, StatusSet.FAILED, StatusSet.SUCCESS], StatusSet.CANCELED: [StatusSet.WAITING], StatusSet.TIMEOUT: [StatusSet.FAILED, StatusSet.SUCCESS, StatusSet.WAITING], - StatusSet.FAILED: [StatusSet.WAITING], + StatusSet.FAILED: [StatusSet.WAITING, StatusSet.CANCELED], StatusSet.SUCCESS: [StatusSet.WAITING], } @@ -83,7 +85,7 @@ class TaskStatus(BaseStatus): class StateTransitionRule(BaseStateTransitionRule): RULES = { - StatusSet.WAITING: [StatusSet.RUNNING, StatusSet.SUCCESS, StatusSet.PASS], + StatusSet.WAITING: [StatusSet.RUNNING, StatusSet.SUCCESS], StatusSet.RUNNING: [StatusSet.CANCELED, StatusSet.TIMEOUT, StatusSet.FAILED, StatusSet.PASS, StatusSet.SUCCESS], StatusSet.CANCELED: [StatusSet.WAITING], StatusSet.TIMEOUT: [StatusSet.FAILED, StatusSet.SUCCESS], @@ -125,29 +127,6 @@ class SuccessStatus(BaseStatus): SUCCESS = StatusSet.SUCCESS -class LinkisJobStatus(BaseStatus): - FAILED = "Failed" - SUCCESS = "Success" - RUNNING = "Running" - - class AutoRerunStatus(BaseStatus): TIMEOUT = StatusSet.TIMEOUT FAILED = StatusSet.FAILED - - -class SchedulingStatusCode(object): - SUCCESS = 0 - NO_RESOURCE = 1 - PASS = 1 - NO_NEXT = 2 - HAVE_NEXT = 3 - FAILED = 4 - - -class FederatedSchedulingStatusCode(object): - SUCCESS = 0 - PARTIAL = 1 - FAILED = 2 - ERROR = 3 - NOT_EFFECTIVE = 4 diff --git a/python/fate_flow/entity/types/_work.py b/python/fate_flow/entity/types/_work.py new file mode 100644 index 000000000..ba2b5da74 --- /dev/null +++ b/python/fate_flow/entity/types/_work.py @@ -0,0 +1,30 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.entity import CustomEnum + + +class ProcessRole(CustomEnum): + DRIVER = "driver" + WORKER = "worker" + + +class WorkerName(CustomEnum): + TASK_SUBMIT = "submit" + TASK_ENTRYPOINT = "task_entrypoint" + TASK_EXECUTE = "task_execute" + COMPONENT_DEFINE = "component_define" + TASK_CLEAN = "task_clean" + TASK_EXECUTE_CLEAN = "execute_clean" diff --git a/python/fate_flow/errors/__init__.py b/python/fate_flow/errors/__init__.py index 3c19241f2..e78b39576 100644 --- a/python/fate_flow/errors/__init__.py +++ b/python/fate_flow/errors/__init__.py @@ -1,10 +1,17 @@ -from .general_error import * - - class FateFlowError(Exception): + code = None message = 'Unknown Fate Flow Error' - def __init__(self, message=None, *args, **kwargs): - message = str(message) if message is not None else self.message - message = message.format(*args, **kwargs) - super().__init__(message) + def __init__(self, message=None, **kwargs): + self.code = self.code + self.message = str(message) if message is not None else self.message + suffix = "" + if kwargs: + for k, v in kwargs.items(): + if v is not None and not callable(v): + if suffix: + suffix += "," + suffix += f"{k}[{v}]" + if suffix: + self.message += f": {suffix}" + super().__init__(self.code, self.message) diff --git a/python/fate_flow/errors/error_checkpoint.py b/python/fate_flow/errors/error_checkpoint.py deleted file mode 100644 index ba90d39bb..000000000 --- a/python/fate_flow/errors/error_checkpoint.py +++ /dev/null @@ -1,7 +0,0 @@ -from fate_flow.errors import FateFlowError - -__all__ = ['CheckpointError'] - - -class CheckpointError(FateFlowError): - message = 'Unknown checkpoint error' diff --git a/python/fate_flow/errors/server_error.py b/python/fate_flow/errors/server_error.py new file mode 100644 index 000000000..ba012b82d --- /dev/null +++ b/python/fate_flow/errors/server_error.py @@ -0,0 +1,157 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.entity.code import ReturnCode +from fate_flow.errors import FateFlowError + + +class JobParamsError(FateFlowError): + code = ReturnCode.Job.PARAMS_ERROR + message = 'Job params error' + + +class NoFoundJob(FateFlowError): + code = ReturnCode.Job.NOT_FOUND + message = 'No found job' + + +class FileNoFound(FateFlowError): + code = ReturnCode.File.FILE_NOT_FOUND + message = 'No found file or dir' + + +class CreateJobFailed(FateFlowError): + code = ReturnCode.Job.CREATE_JOB_FAILED + message = 'Create job failed' + + +class UpdateJobFailed(FateFlowError): + code = ReturnCode.Job.UPDATE_FAILED + message = 'Update job does not take effect' + + +class KillFailed(FateFlowError): + code = ReturnCode.Job.KILL_FAILED + message = "Kill job failed" + + +class JobResourceException(FateFlowError): + code = ReturnCode.Job.RESOURCE_EXCEPTION + message = "Job resource exception" + + +class InheritanceFailed(FateFlowError): + code = ReturnCode.Job.INHERITANCE_FAILED + message = "Inheritance job failed" + + +class NoFoundTask(FateFlowError): + code = ReturnCode.Task.NOT_FOUND + message = "No found task" + + +class StartTaskFailed(FateFlowError): + code = ReturnCode.Task.START_FAILED + message = "Start task failed" + + +class UpdateTaskFailed(FateFlowError): + code = ReturnCode.Task.UPDATE_FAILED + message = "Update task status does not take effect" + + +class KillTaskFailed(FateFlowError): + code = ReturnCode.Task.KILL_FAILED + message = 'Kill task failed' + + +class TaskResourceException(FateFlowError): + code = ReturnCode.Task.RESOURCE_EXCEPTION + message = "Task resource exception" + + +class NoFoundModelOutput(FateFlowError): + code = ReturnCode.Task.NO_FOUND_MODEL_OUTPUT + message = "No found output model" + + +class IsStandalone(FateFlowError): + code = ReturnCode.Site.IS_STANDALONE + message = "Site is standalone" + + +class DeviceNotSupported(FateFlowError): + code = ReturnCode.Provider.DEVICE_NOT_SUPPORTED + message = "Device not supported" + + +class RequestExpired(FateFlowError): + code = ReturnCode.API.EXPIRED + message = "Request has expired" + + +class InvalidParameter(FateFlowError): + code = ReturnCode.API.INVALID_PARAMETER + message = "Invalid parameter" + + +class NoFoundAppid(FateFlowError): + code = ReturnCode.API.NO_FOUND_APPID + message = "No found appid" + + +class ResponseException(FateFlowError): + code = ReturnCode.Server.RESPONSE_EXCEPTION + message = "Response exception" + + +class NoFoundServer(FateFlowError): + code = ReturnCode.Server.NO_FOUND + message = "No found server" + + +class NoFoundINSTANCE(FateFlowError): + code = ReturnCode.Server.NO_FOUND_INSTANCE + message = "No Found Flow Instance" + + +class NoFoundTable(FateFlowError): + code = ReturnCode.Table.NO_FOUND + message = "No found table" + + +class ExistsTable(FateFlowError): + code = ReturnCode.Table.EXISTS + message = "Exists table" + + +class NoPermission(FateFlowError): + code = ReturnCode.API.NO_PERMISSION + message = "No Permission" + + +class PermissionOperateError(FateFlowError): + code = ReturnCode.API.PERMISSION_OPERATE_ERROR + message = "Permission Operate Error" + + +class NoFoundFile(FateFlowError): + code = ReturnCode.API.NO_FOUND_FILE + message = "No Found File" + + +class RoleTypeError(FateFlowError): + code = ReturnCode.API.ROLE_TYPE_ERROR + message = "Role Type Error" diff --git a/python/fate_flow/errors/error_services.py b/python/fate_flow/errors/zookeeper_error.py similarity index 100% rename from python/fate_flow/errors/error_services.py rename to python/fate_flow/errors/zookeeper_error.py diff --git a/python/fate_flow/external/data_storage.py b/python/fate_flow/external/data_storage.py deleted file mode 100644 index b6f4375ef..000000000 --- a/python/fate_flow/external/data_storage.py +++ /dev/null @@ -1,10 +0,0 @@ -from fate_flow.entity.types import ExternalStorage -from fate_flow.external.storage import MysqlStorage - - -def save_data_to_external_storage(storage_engine, address, storage_table): - if storage_engine.upper() in {ExternalStorage.MYSQL.value}: - with MysqlStorage(address=address, storage_table=storage_table) as storage: - storage.save() - else: - raise ValueError(f"{storage_engine.upper()} is not supported") diff --git a/python/fate_flow/external/storage/base.py b/python/fate_flow/external/storage/base.py deleted file mode 100644 index 7791a83a6..000000000 --- a/python/fate_flow/external/storage/base.py +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import abc - - -class AddressABC(metaclass=abc.ABCMeta): - ... - - -class Storage(metaclass=abc.ABCMeta): - @abc.abstractmethod - def save(self): - ... - - -class MysqlAddress(AddressABC): - def __init__(self, user, passwd, host, port, db, name, **kwargs): - self.user = user - self.passwd = passwd - self.host = host - self.port = port - self.db = db - self.name = name - - def __hash__(self): - return (self.host, self.port, self.db, self.name).__hash__() - - def __str__(self): - return f"MysqlAddress(db={self.db}, name={self.name})" - - def __repr__(self): - return self.__str__() diff --git a/python/fate_flow/external/storage/mysql.py b/python/fate_flow/external/storage/mysql.py deleted file mode 100644 index fff456af9..000000000 --- a/python/fate_flow/external/storage/mysql.py +++ /dev/null @@ -1,124 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import traceback - -import pymysql - -from fate_arch.common import log -from fate_arch.storage import StorageTableBase -from fate_flow.component_env_utils import feature_utils -from fate_flow.external.storage.base import Storage, MysqlAddress -from fate_flow.manager.data_manager import get_component_output_data_schema - -LOGGER = log.getLogger() - - -class MysqlStorage(Storage): - def __init__(self, address: dict, storage_table: StorageTableBase): - self.address = MysqlAddress(**address) - self.storage_table = storage_table - self._con = None - self._cur = None - self._connect() - - def save(self): - create = False - sql = None - max = 10000 - count = 0 - LOGGER.info(f"start save Table({self.storage_table.namespace}, {self.storage_table.name}) to Mysql({self.address.db}, {self.address.name})") - join_delimiter = "," - for k, v in self.storage_table.collect(): - v, extend_header = feature_utils.get_deserialize_value(v, join_delimiter) - if not create: - _, header_list = self._create_table(extend_header) - LOGGER.info("craete table success") - create = True - if not sql: - sql = "REPLACE INTO {}({}, {}) VALUES".format( - self.address.name, header_list[0], ",".join(header_list[1:]) - ) - sql += '("{}", "{}"),'.format(k, '", "'.join(v.split(join_delimiter))) - count += 1 - if not count % max: - sql = ",".join(sql.split(",")[:-1]) + ";" - self._cur.execute(sql) - self._con.commit() - sql = None - LOGGER.info(f"save data count:{count}") - if count > 0 and sql: - sql = ",".join(sql.split(",")[:-1]) + ";" - self._cur.execute(sql) - self._con.commit() - LOGGER.info(f"save success, count:{count}") - - def _create_table(self, extend_header): - header_list = get_component_output_data_schema(self.storage_table.meta, extend_header) - feature_sql = self.get_create_features_sql(header_list[1:]) - id_size = "varchar(100)" - create_table = ( - "create table if not exists {}({} {} NOT NULL, {} PRIMARY KEY({}))".format( - self.address.name, header_list[0], id_size, feature_sql, header_list[0] - ) - ) - LOGGER.info(f"create table {self.address.name}: {create_table}") - return self._cur.execute(create_table), header_list - - @staticmethod - def get_create_features_sql(feature_name_list): - create_features = "" - feature_list = [] - feature_size = "varchar(255)" - for feature_name in feature_name_list: - create_features += "{} {},".format(feature_name, feature_size) - feature_list.append(feature_name) - return create_features - - def _create_db_if_not_exists(self): - connection = pymysql.connect(host=self.address.host, - user=self.address.user, - password=self.address.passwd, - port=self.address.port) - with connection: - with connection.cursor() as cursor: - cursor.execute("create database if not exists {}".format(self.address.db)) - print('create db {} success'.format(self.address.db)) - connection.commit() - - def _connect(self): - LOGGER.info(f"start connect database {self.address.db}") - self._con = pymysql.connect(host=self.address.host, - user=self.address.user, - passwd=self.address.passwd, - port=self.address.port, - db=self.address.db) - self._cur = self._con.cursor() - LOGGER.info(f"connect success!") - - def _open(self): - return self - - def __enter__(self): - self._connect() - return self._open() - - def __exit__(self, exc_type, exc_val, exc_tb): - try: - LOGGER.info("close connect") - self._cur.close() - self._con.close() - except Exception as e: - traceback.print_exc() \ No newline at end of file diff --git a/python/fate_flow/fate_flow_client.py b/python/fate_flow/fate_flow_client.py deleted file mode 100644 index d401114ed..000000000 --- a/python/fate_flow/fate_flow_client.py +++ /dev/null @@ -1,298 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import sys -import argparse -import json -import os -import tarfile -import traceback -from contextlib import closing -import time -import re -import requests -from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor -# be sure to import environment variable before importing fate_arch -from fate_flow import set_env -from fate_arch.common import file_utils -from fate_flow.settings import API_VERSION, HOST, HTTP_PORT -from fate_flow.utils import detect_utils, requests_utils -from fate_flow.utils.base_utils import get_fate_flow_directory - -JOB_OPERATE_FUNC = ["submit_job", "stop_job", "query_job", "data_view_query", "clean_job", "clean_queue"] -JOB_FUNC = ["job_config", "job_log_download"] -TASK_OPERATE_FUNC = ["query_task"] -TRACKING_FUNC = ["component_parameters", "component_metric_all", "component_metric_delete", "component_metrics", - "component_output_model", "component_output_data", "component_output_data_table"] -DATA_FUNC = ["download", "upload", "upload_history"] -TABLE_FUNC = ["table_info", "table_delete", "table_add", "table_bind"] -MODEL_FUNC = ["load", "bind", "store", "restore", "export", "import"] -PERMISSION_FUNC = ["grant_privilege", "delete_privilege", "query_privilege"] - - -def prettify(response, verbose=True): - if verbose: - print(json.dumps(response, indent=4, ensure_ascii=False)) - print() - return response - - -def call_fun(func, config_data, dsl_path, config_path): - server_url = "http://{}:{}/{}".format(HOST, HTTP_PORT, API_VERSION) - response = None - - if func in JOB_OPERATE_FUNC: - if func == 'submit_job': - if not config_path: - raise Exception('the following arguments are required: {}'.format('runtime conf path')) - if not dsl_path and config_data.get('job_parameters', {}).get('job_type', '') == 'predict': - raise Exception('for train job, the following arguments are required: {}'.format('dsl path')) - dsl_data = {} - if dsl_path: - dsl_path = os.path.abspath(dsl_path) - with open(dsl_path, 'r') as f: - dsl_data = json.load(f) - post_data = {'job_dsl': dsl_data, - 'job_runtime_conf': config_data} - response = requests_utils.request(method="post", url="/".join([server_url, "job", func.rstrip('_job')]), json=post_data) - try: - if response.json()['retcode'] == 999: - start_cluster_standalone_job_server() - response = requests_utils.request(method="post", url="/".join([server_url, "job", func.rstrip('_job')]), json=post_data) - except: - pass - elif func == 'data_view_query' or func == 'clean_queue': - response = requests_utils.request(method="post", url="/".join([server_url, "job", func.replace('_', '/')]), json=config_data) - else: - if func != 'query_job': - detect_utils.check_config(config=config_data, required_arguments=['job_id']) - post_data = config_data - response = requests_utils.request(method="post", url="/".join([server_url, "job", func.rstrip('_job')]), json=post_data) - if func == 'query_job': - response = response.json() - if response['retcode'] == 0: - for i in range(len(response['data'])): - del response['data'][i]['f_runtime_conf'] - del response['data'][i]['f_dsl'] - elif func in JOB_FUNC: - if func == 'job_config': - detect_utils.check_config(config=config_data, required_arguments=['job_id', 'role', 'party_id', 'output_path']) - response = requests_utils.request(method="post", url="/".join([server_url, func.replace('_', '/')]), json=config_data) - response_data = response.json() - if response_data['retcode'] == 0: - job_id = response_data['data']['job_id'] - download_directory = os.path.join(config_data['output_path'], 'job_{}_config'.format(job_id)) - os.makedirs(download_directory, exist_ok=True) - for k, v in response_data['data'].items(): - if k == 'job_id': - continue - with open('{}/{}.json'.format(download_directory, k), 'w') as fw: - json.dump(v, fw, indent=4) - del response_data['data']['dsl'] - del response_data['data']['runtime_conf'] - response_data['directory'] = download_directory - response_data['retmsg'] = 'download successfully, please check {} directory'.format(download_directory) - response = response_data - elif func == 'job_log_download': - detect_utils.check_config(config=config_data, required_arguments=['job_id', 'output_path']) - job_id = config_data['job_id'] - tar_file_name = 'job_{}_log.tar.gz'.format(job_id) - extract_dir = os.path.join(config_data['output_path'], 'job_{}_log'.format(job_id)) - with closing(requests_utils.request(method="post", url="/".join([server_url, func.replace('_', '/')]), json=config_data, - stream=True)) as response: - if response.status_code == 200: - download_from_request(http_response=response, tar_file_name=tar_file_name, extract_dir=extract_dir) - response = {'retcode': 0, - 'directory': extract_dir, - 'retmsg': 'download successfully, please check {} directory'.format(extract_dir)} - else: - response = response.json() - elif func in TASK_OPERATE_FUNC: - response = requests_utils.request(method="post", url="/".join([server_url, "job", "task", func.rstrip('_task')]), json=config_data) - elif func in TRACKING_FUNC: - if func != 'component_metric_delete': - detect_utils.check_config(config=config_data, - required_arguments=['job_id', 'component_name', 'role', 'party_id']) - if func == 'component_output_data': - detect_utils.check_config(config=config_data, required_arguments=['output_path']) - tar_file_name = 'job_{}_{}_{}_{}_output_data.tar.gz'.format(config_data['job_id'], - config_data['component_name'], - config_data['role'], - config_data['party_id']) - extract_dir = os.path.join(config_data['output_path'], tar_file_name.replace('.tar.gz', '')) - with closing(requests_utils.request(method="get", url="/".join([server_url, "tracking", func.replace('_', '/'), 'download']), - json=config_data, stream=True)) as response: - if response.status_code == 200: - try: - download_from_request(http_response=response, tar_file_name=tar_file_name, extract_dir=extract_dir) - response = {'retcode': 0, - 'directory': extract_dir, - 'retmsg': 'download successfully, please check {} directory'.format(extract_dir)} - except: - response = {'retcode': 100, - 'retmsg': 'download failed, please check if the parameters are correct'} - else: - response = response.json() - - else: - response = requests_utils.request(method="post", url="/".join([server_url, "tracking", func.replace('_', '/')]), json=config_data) - elif func in DATA_FUNC: - if func == 'upload' and config_data.get('use_local_data', 1) != 0: - file_name = config_data.get('file') - if not os.path.isabs(file_name): - file_name = os.path.join(get_fate_flow_directory(), file_name) - if os.path.exists(file_name): - with open(file_name, 'rb') as fp: - data = MultipartEncoder( - fields={'file': (os.path.basename(file_name), fp, 'application/octet-stream')} - ) - tag = [0] - - def read_callback(monitor): - if config_data.get('verbose') == 1: - sys.stdout.write("\r UPLOADING:{0}{1}".format("|" * (monitor.bytes_read * 100 // monitor.len), '%.2f%%' % (monitor.bytes_read * 100 // monitor.len))) - sys.stdout.flush() - if monitor.bytes_read /monitor.len == 1: - tag[0] += 1 - if tag[0] == 2: - sys.stdout.write('\n') - data = MultipartEncoderMonitor(data, read_callback) - response = requests_utils.request(method="post", url="/".join([server_url, "data", func.replace('_', '/')]), data=data, - params=json.dumps(config_data), headers={'Content-Type': data.content_type}) - else: - raise Exception('The file is obtained from the fate flow client machine, but it does not exist, ' - 'please check the path: {}'.format(file_name)) - else: - response = requests_utils.request(method="post", url="/".join([server_url, "data", func.replace('_', '/')]), json=config_data) - try: - if response.json()['retcode'] == 999: - start_cluster_standalone_job_server() - response = requests_utils.request(method="post", url="/".join([server_url, "data", func]), json=config_data) - except: - pass - elif func in TABLE_FUNC: - if func == "table_info": - detect_utils.check_config(config=config_data, required_arguments=['namespace', 'table_name']) - response = requests_utils.request(method="post", url="/".join([server_url, "table", func]), json=config_data) - else: - response = requests_utils.request(method="post", url="/".join([server_url, func.replace('_', '/')]), json=config_data) - elif func in MODEL_FUNC: - if func == "import": - file_path = config_data["file"] - if not os.path.isabs(file_path): - file_path = os.path.join(get_fate_flow_directory(), file_path) - if os.path.exists(file_path): - files = {'file': open(file_path, 'rb')} - else: - raise Exception('The file is obtained from the fate flow client machine, but it does not exist, ' - 'please check the path: {}'.format(file_path)) - response = requests_utils.request(method="post", url="/".join([server_url, "model", func]), data=config_data, files=files) - elif func == "export": - with closing(requests_utils.request(method="get", url="/".join([server_url, "model", func]), json=config_data, stream=True)) as response: - if response.status_code == 200: - archive_file_name = re.findall("filename=(.+)", response.headers["Content-Disposition"])[0] - os.makedirs(config_data["output_path"], exist_ok=True) - archive_file_path = os.path.join(config_data["output_path"], archive_file_name) - with open(archive_file_path, 'wb') as fw: - for chunk in response.iter_content(1024): - if chunk: - fw.write(chunk) - response = {'retcode': 0, - 'file': archive_file_path, - 'retmsg': 'download successfully, please check {}'.format(archive_file_path)} - else: - response = response.json() - else: - response = requests_utils.request(method="post", url="/".join([server_url, "model", func]), json=config_data) - elif func in PERMISSION_FUNC: - detect_utils.check_config(config=config_data, required_arguments=['src_party_id', 'src_role']) - response = requests_utils.request(method="post", url="/".join([server_url, "permission", func.replace('_', '/')]), json=config_data) - return response.json() if isinstance(response, requests.models.Response) else response - - -def download_from_request(http_response, tar_file_name, extract_dir): - with open(tar_file_name, 'wb') as fw: - for chunk in http_response.iter_content(1024): - if chunk: - fw.write(chunk) - tar = tarfile.open(tar_file_name, "r:gz") - file_names = tar.getnames() - for file_name in file_names: - tar.extract(file_name, extract_dir) - tar.close() - os.remove(tar_file_name) - - -def start_cluster_standalone_job_server(): - print('use service.sh to start standalone node server....') - os.system('sh service.sh start --standalone_node') - time.sleep(5) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('-c', '--config', required=False, type=str, help="runtime conf path") - parser.add_argument('-d', '--dsl', required=False, type=str, help="dsl path") - parser.add_argument('-f', '--function', type=str, - choices=( - DATA_FUNC + MODEL_FUNC + JOB_FUNC + JOB_OPERATE_FUNC + TASK_OPERATE_FUNC + TABLE_FUNC + - TRACKING_FUNC + PERMISSION_FUNC), - required=True, - help="function to call") - parser.add_argument('-j', '--job_id', required=False, type=str, help="job id") - parser.add_argument('-p', '--party_id', required=False, type=str, help="party id") - parser.add_argument('-r', '--role', required=False, type=str, help="role") - parser.add_argument('-cpn', '--component_name', required=False, type=str, help="component name") - parser.add_argument('-s', '--status', required=False, type=str, help="status") - parser.add_argument('-n', '--namespace', required=False, type=str, help="namespace") - parser.add_argument('-t', '--table_name', required=False, type=str, help="table name") - parser.add_argument('-w', '--work_mode', required=False, type=int, help="work mode") - parser.add_argument('-i', '--file', required=False, type=str, help="file") - parser.add_argument('-o', '--output_path', required=False, type=str, help="output_path") - parser.add_argument('-m', '--model', required=False, type=str, help="TrackingMetric model id") - parser.add_argument('-drop', '--drop', required=False, type=str, help="drop data table") - parser.add_argument('-limit', '--limit', required=False, type=int, help="limit number") - parser.add_argument('-verbose', '--verbose', required=False, type=int, help="number 0 or 1") - parser.add_argument('-src_party_id', '--src_party_id', required=False, type=str, help="src party id") - parser.add_argument('-src_role', '--src_role', required=False, type=str, help="src role") - parser.add_argument('-privilege_role', '--privilege_role', required=False, type=str, help="privilege role") - parser.add_argument('-privilege_command', '--privilege_command', required=False, type=str, help="privilege command") - parser.add_argument('-privilege_component', '--privilege_component', required=False, type=str, help="privilege component") - try: - args = parser.parse_args() - config_data = {} - dsl_path = args.dsl - config_path = args.config - if args.config: - args.config = os.path.abspath(args.config) - with open(args.config, 'r') as f: - config_data = json.load(f) - config_data.update(dict((k, v) for k, v in vars(args).items() if v is not None)) - if args.party_id or args.role: - config_data['local'] = config_data.get('local', {}) - if args.party_id: - config_data['local']['party_id'] = args.party_id - if args.role: - config_data['local']['role'] = args.role - if config_data.get('output_path'): - config_data['output_path'] = os.path.abspath(config_data["output_path"]) - response = call_fun(args.function, config_data, dsl_path, config_path) - except Exception as e: - exc_type, exc_value, exc_traceback_obj = sys.exc_info() - response = {'retcode': 100, 'retmsg': str(e), 'traceback': traceback.format_exception(exc_type, exc_value, exc_traceback_obj)} - if 'Connection refused' in str(e): - response['retmsg'] = 'Connection refused, Please check if the fate flow service is started' - del response['traceback'] - response_dict = prettify(response) diff --git a/python/fate_flow/fate_flow_server.py b/python/fate_flow/fate_flow_server.py index d64db9810..b4a4363b7 100644 --- a/python/fate_flow/fate_flow_server.py +++ b/python/fate_flow/fate_flow_server.py @@ -13,10 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# init env. must be the first import -import fate_flow as _ - -import logging import os import signal import sys @@ -25,101 +21,120 @@ import grpc from werkzeug.serving import run_simple -from fate_arch.common import file_utils -from fate_arch.common.versions import get_versions -from fate_arch.metastore.db_models import init_database_tables as init_arch_db -from fate_arch.protobuf.python import proxy_pb2_grpc +if __name__ == '__main__': + from fate_flow.db.casbin_models import init_casbin + init_casbin() from fate_flow.apps import app -from fate_flow.controller.version_controller import VersionController -from fate_flow.db.component_registry import ComponentRegistry -from fate_flow.db.config_manager import ConfigManager -from fate_flow.db.db_models import init_database_tables as init_flow_db -from fate_flow.db.db_services import service_db -from fate_flow.db.key_manager import RsaKeyManager -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.detection.detector import Detector, FederatedDetector -from fate_flow.entity.types import ProcessRole +from fate_flow.manager.service.config_manager import ConfigManager from fate_flow.hook import HookManager -from fate_flow.manager.provider_manager import ProviderManager -from fate_flow.scheduler.dag_scheduler import DAGScheduler -from fate_flow.settings import ( - GRPC_OPTIONS, GRPC_PORT, GRPC_SERVER_MAX_WORKERS, HOST, HTTP_PORT, - access_logger, database_logger, detect_logger, stat_logger, +from fate_flow.manager.service.app_manager import AppManager +from fate_flow.manager.service.provider_manager import ProviderManager +from fate_flow.manager.service.service_manager import service_db +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.db.base_models import init_database_tables as init_flow_db +from fate_flow.scheduler.detector import Detector, FederatedDetector +from fate_flow.entity.types import ProcessRole +from fate_flow.scheduler import init_scheduler +from fate_flow.runtime.system_settings import ( + GRPC_PORT, GRPC_SERVER_MAX_WORKERS, HOST, HTTP_PORT , GRPC_OPTIONS, FATE_FLOW_LOG_DIR, + LOG_LEVEL, ) -from fate_flow.utils.base_utils import get_fate_flow_directory +from fate_flow.scheduler.scheduler import DAGScheduler +from fate_flow.utils import process_utils from fate_flow.utils.grpc_utils import UnaryService +from fate_flow.utils.log import LoggerFactory, getLogger from fate_flow.utils.log_utils import schedule_logger +from fate_flow.utils.version import get_versions from fate_flow.utils.xthread import ThreadPoolExecutor +from fate_flow.proto.rollsite import proxy_pb2_grpc +detect_logger = getLogger("fate_flow_detect") +stat_logger = getLogger("fate_flow_stat") -if __name__ == '__main__': - stat_logger.info( - f'project base: {file_utils.get_project_base_directory()}, ' - f'fate base: {file_utils.get_fate_directory()}, ' - f'fate flow base: {get_fate_flow_directory()}' - ) + +def server_init(): + # init logs + LoggerFactory.set_directory(FATE_FLOW_LOG_DIR) + LoggerFactory.LEVEL = LOG_LEVEL + + # set signal + if "win" not in sys.platform.lower(): + signal.signal(signal.SIGCHLD, process_utils.wait_child_process) + + # init adapter + try: + from fate_flow.adapter import init_adapter + init_adapter() + except Exception as ex: + stat_logger.exception(ex) # init db init_flow_db() - init_arch_db() - # init runtime config - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--version', default=False, help="fate flow version", action='store_true') - parser.add_argument('--debug', default=False, help="debug mode", action='store_true') - args = parser.parse_args() - if args.version: - print(get_versions()) - sys.exit(0) - # todo: add a general init steps? - RuntimeConfig.DEBUG = args.debug - if RuntimeConfig.DEBUG: - stat_logger.info("run on debug mode") - ConfigManager.load() + + # runtime config RuntimeConfig.init_env() RuntimeConfig.init_config(JOB_SERVER_HOST=HOST, HTTP_PORT=HTTP_PORT) - RuntimeConfig.set_process_role(ProcessRole.DRIVER) - + RuntimeConfig.init_config() RuntimeConfig.set_service_db(service_db()) RuntimeConfig.SERVICE_DB.register_flow() - RuntimeConfig.SERVICE_DB.register_models() - ComponentRegistry.load() - default_algorithm_provider = ProviderManager.register_default_providers() - RuntimeConfig.set_component_provider(default_algorithm_provider) - ComponentRegistry.load() + # manager + ConfigManager.load() HookManager.init() - RsaKeyManager.init() - VersionController.init() + AppManager.init() + + # scheduler + init_scheduler() + + # detector Detector(interval=5 * 1000, logger=detect_logger).start() FederatedDetector(interval=10 * 1000, logger=detect_logger).start() DAGScheduler(interval=2 * 1000, logger=schedule_logger()).start() - peewee_logger = logging.getLogger('peewee') - peewee_logger.propagate = False - # fate_arch.common.log.ROpenHandler - peewee_logger.addHandler(database_logger.handlers[0]) - peewee_logger.setLevel(database_logger.level) + # provider register + ProviderManager.register_default_providers() + +def start_server(debug=False): + # grpc thread_pool_executor = ThreadPoolExecutor(max_workers=GRPC_SERVER_MAX_WORKERS) - stat_logger.info(f"start grpc server thread pool by {thread_pool_executor._max_workers} max workers") - server = grpc.server(thread_pool=thread_pool_executor, options=GRPC_OPTIONS) + stat_logger.info(f"start grpc server thread pool by {thread_pool_executor.max_workers} max workers") + server = grpc.server(thread_pool=thread_pool_executor, + options=GRPC_OPTIONS) proxy_pb2_grpc.add_DataTransferServiceServicer_to_server(UnaryService(), server) server.add_insecure_port(f"{HOST}:{GRPC_PORT}") server.start() - print("FATE Flow grpc server start successfully") stat_logger.info("FATE Flow grpc server start successfully") - # start http server + # http + stat_logger.info("FATE Flow http server start...") + run_simple( + hostname=HOST, + port=HTTP_PORT, + application=app, + threaded=True, + use_reloader=debug, + use_debugger=debug + ) + + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--version', default=False, help="fate flow version", action='store_true') + parser.add_argument('--debug', default=False, help="debug mode", action='store_true') + args = parser.parse_args() + if args.version: + print(get_versions()) + sys.exit(0) + + server_init() + try: - print("FATE Flow http server start...") - stat_logger.info("FATE Flow http server start...") - werkzeug_logger = logging.getLogger("werkzeug") - for h in access_logger.handlers: - werkzeug_logger.addHandler(h) - run_simple(hostname=HOST, port=HTTP_PORT, application=app, threaded=True, use_reloader=RuntimeConfig.DEBUG, use_debugger=RuntimeConfig.DEBUG) - except Exception: + start_server(debug=args.debug) + except Exception as e: traceback.print_exc() + print(e) os.kill(os.getpid(), signal.SIGKILL) diff --git a/python/fate_flow/hook/__init__.py b/python/fate_flow/hook/__init__.py index b6a80f082..99abe9aef 100644 --- a/python/fate_flow/hook/__init__.py +++ b/python/fate_flow/hook/__init__.py @@ -1,9 +1,12 @@ import importlib from fate_flow.hook.common.parameters import SignatureParameters, AuthenticationParameters, PermissionCheckParameters, \ - SignatureReturn, AuthenticationReturn, PermissionReturn, ClientAuthenticationReturn, ClientAuthenticationParameters -from fate_flow.settings import HOOK_MODULE, stat_logger -from fate_flow.entity import RetCode + SignatureReturn, AuthenticationReturn, PermissionReturn +from fate_flow.runtime.system_settings import HOOK_MODULE, CLIENT_AUTHENTICATION, SITE_AUTHENTICATION, PERMISSION_SWITCH +from fate_flow.entity.code import ReturnCode +from fate_flow.utils.log import getLogger + +stat_logger = getLogger() class HookManager: @@ -14,7 +17,7 @@ class HookManager: @staticmethod def init(): - if HOOK_MODULE is not None: + if HOOK_MODULE is not None and (CLIENT_AUTHENTICATION or SITE_AUTHENTICATION or PERMISSION_SWITCH): for modules in HOOK_MODULE.values(): for module in modules.split(";"): try: @@ -24,7 +27,8 @@ def init(): @staticmethod def register_site_signature_hook(func): - HookManager.SITE_SIGNATURE.append(func) + if SITE_AUTHENTICATION: + HookManager.SITE_SIGNATURE.append(func) @staticmethod def register_site_authentication_hook(func): @@ -39,10 +43,10 @@ def register_permission_check_hook(func): HookManager.PERMISSION_CHECK.append(func) @staticmethod - def client_authentication(parm: ClientAuthenticationParameters) -> ClientAuthenticationReturn: + def client_authentication(parm: AuthenticationParameters) -> AuthenticationReturn: if HookManager.CLIENT_AUTHENTICATION: return HookManager.CLIENT_AUTHENTICATION[0](parm) - return ClientAuthenticationReturn() + return AuthenticationReturn() @staticmethod def site_signature(parm: SignatureParameters) -> SignatureReturn: @@ -59,8 +63,5 @@ def site_authentication(parm: AuthenticationParameters) -> AuthenticationReturn: @staticmethod def permission_check(parm: PermissionCheckParameters) -> PermissionReturn: if HookManager.PERMISSION_CHECK: - for permission_check_func in HookManager.PERMISSION_CHECK: - result = permission_check_func(parm) - if result.code != RetCode.SUCCESS: - return result + return HookManager.PERMISSION_CHECK[0](parm) return PermissionReturn() diff --git a/python/fate_flow/hook/api/client_authentication.py b/python/fate_flow/hook/api/client_authentication.py deleted file mode 100644 index f0ea1c073..000000000 --- a/python/fate_flow/hook/api/client_authentication.py +++ /dev/null @@ -1,29 +0,0 @@ -import requests - -from fate_flow.db.service_registry import ServiceRegistry -from fate_flow.entity.types import RegistryServiceName -from fate_flow.hook import HookManager -from fate_flow.hook.common.parameters import ClientAuthenticationParameters, ClientAuthenticationReturn -from fate_flow.settings import HOOK_SERVER_NAME - - -@HookManager.register_client_authentication_hook -def authentication(parm: ClientAuthenticationParameters) -> ClientAuthenticationReturn: - service_list = ServiceRegistry.load_service( - server_name=HOOK_SERVER_NAME, - service_name=RegistryServiceName.CLIENT_AUTHENTICATION.value - ) - if not service_list: - raise Exception(f"client authentication error: no found server" - f" {HOOK_SERVER_NAME} service client_authentication") - service = service_list[0] - response = getattr(requests, service.f_method.lower(), None)( - url=service.f_url, - json=parm.to_dict() - ) - if response.status_code != 200: - raise Exception( - f"client authentication error: request authentication url failed, status code {response.status_code}") - elif response.json().get("code") != 0: - return ClientAuthenticationReturn(code=response.json().get("code"), message=response.json().get("msg")) - return ClientAuthenticationReturn() \ No newline at end of file diff --git a/python/fate_flow/hook/api/permission.py b/python/fate_flow/hook/api/permission.py deleted file mode 100644 index 688089262..000000000 --- a/python/fate_flow/hook/api/permission.py +++ /dev/null @@ -1,25 +0,0 @@ -import requests - -from fate_flow.db.service_registry import ServiceRegistry -from fate_flow.entity.types import RegistryServiceName -from fate_flow.hook import HookManager -from fate_flow.hook.common.parameters import PermissionCheckParameters, PermissionReturn -from fate_flow.settings import HOOK_SERVER_NAME - - -@HookManager.register_permission_check_hook -def permission(parm: PermissionCheckParameters) -> PermissionReturn: - service_list = ServiceRegistry.load_service(server_name=HOOK_SERVER_NAME, service_name=RegistryServiceName.PERMISSION_CHECK.value) - if not service_list: - raise Exception(f"permission check error: no found server {HOOK_SERVER_NAME} service permission") - service = service_list[0] - response = getattr(requests, service.f_method.lower(), None)( - url=service.f_url, - json=parm.to_dict() - ) - if response.status_code != 200: - raise Exception( - f"permission check error: request permission url failed, status code {response.status_code}") - elif response.json().get("code") != 0: - return PermissionReturn(code=response.json().get("code"), message=response.json().get("msg")) - return PermissionReturn() diff --git a/python/fate_flow/hook/api/site_authentication.py b/python/fate_flow/hook/api/site_authentication.py deleted file mode 100644 index 7e36fbe50..000000000 --- a/python/fate_flow/hook/api/site_authentication.py +++ /dev/null @@ -1,53 +0,0 @@ -import requests - -from fate_flow.db.service_registry import ServiceRegistry -from fate_flow.entity.types import RegistryServiceName -from fate_flow.hook import HookManager -from fate_flow.hook.common.parameters import SignatureParameters, AuthenticationParameters, AuthenticationReturn,\ - SignatureReturn -from fate_flow.settings import HOOK_SERVER_NAME, PARTY_ID - - -@HookManager.register_site_signature_hook -def signature(parm: SignatureParameters) -> SignatureReturn: - service_list = ServiceRegistry.load_service(server_name=HOOK_SERVER_NAME, service_name=RegistryServiceName.SIGNATURE.value) - if not service_list: - raise Exception(f"signature error: no found server {HOOK_SERVER_NAME} service signature") - service = service_list[0] - data = service.f_data if service.f_data else {} - data.update(parm.to_dict()) - response = getattr(requests, service.f_method.lower(), None)( - url=service.f_url, - json=data - ) - if response.status_code == 200: - if response.json().get("code") == 0: - return SignatureReturn(site_signature=response.json().get("data")) - else: - raise Exception(f"signature error: request signature url failed, result: {response.json()}") - else: - raise Exception(f"signature error: request signature url failed, status code {response.status_code}") - - -@HookManager.register_site_authentication_hook -def authentication(parm: AuthenticationParameters) -> AuthenticationReturn: - if not parm.src_party_id or str(parm.src_party_id) == "0": - parm.src_party_id = PARTY_ID - service_list = ServiceRegistry.load_service(server_name=HOOK_SERVER_NAME, - service_name=RegistryServiceName.SITE_AUTHENTICATION.value) - if not service_list: - raise Exception( - f"site authentication error: no found server {HOOK_SERVER_NAME} service site_authentication") - service = service_list[0] - data = service.f_data if service.f_data else {} - data.update(parm.to_dict()) - response = getattr(requests, service.f_method.lower(), None)( - url=service.f_url, - json=data - ) - if response.status_code != 200: - raise Exception( - f"site authentication error: request site_authentication url failed, status code {response.status_code}") - elif response.json().get("code") != 0: - return AuthenticationReturn(code=response.json().get("code"), message=response.json().get("msg")) - return AuthenticationReturn() \ No newline at end of file diff --git a/python/fate_flow/hook/common/__init__.py b/python/fate_flow/hook/common/__init__.py new file mode 100644 index 000000000..55fbe5eef --- /dev/null +++ b/python/fate_flow/hook/common/__init__.py @@ -0,0 +1,15 @@ + +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/hook/common/parameters.py b/python/fate_flow/hook/common/parameters.py index 2c22dcbd7..db619f5bd 100644 --- a/python/fate_flow/hook/common/parameters.py +++ b/python/fate_flow/hook/common/parameters.py @@ -1,4 +1,4 @@ -from fate_flow.entity import RetCode +from fate_flow.entity.code import ReturnCode class ParametersBase: @@ -9,63 +9,49 @@ def to_dict(self): return d -class ClientAuthenticationParameters(ParametersBase): - def __init__(self, full_path, headers, form, data, json): - self.full_path = full_path +class AuthenticationParameters(ParametersBase): + def __init__(self, path, method, headers, form, data, json, full_path): + self.path = path + self.method = method self.headers = headers self.form = form self.data = data self.json = json + self.full_path = full_path -class ClientAuthenticationReturn(ParametersBase): - def __init__(self, code=RetCode.SUCCESS, message="success"): +class AuthenticationReturn(ParametersBase): + def __init__(self, code=ReturnCode.Base.SUCCESS, message="success"): self.code = code self.message = message class SignatureParameters(ParametersBase): - def __init__(self, party_id, body): + def __init__(self, party_id, body, initiator_party_id=""): self.party_id = party_id + self.initiator_party_id = initiator_party_id self.body = body class SignatureReturn(ParametersBase): - def __init__(self, code=RetCode.SUCCESS, site_signature=None): - self.code = code - self.site_signature = site_signature - - -class AuthenticationParameters(ParametersBase): - def __init__(self, src_party_id, site_signature, body): - self.src_party_id = src_party_id - self.site_signature = site_signature - self.body = body - - -class AuthenticationReturn(ParametersBase): - def __init__(self, code=RetCode.SUCCESS, message="success"): + def __init__(self, code=ReturnCode.Base.SUCCESS, signature=None, message=""): self.code = code + self.signature = signature self.message = message class PermissionCheckParameters(ParametersBase): - def __init__(self, src_role, src_party_id, role, party_id, initiator, roles, component_list, dataset_list, runtime_conf, dsl, component_parameters): - self.src_role = src_role - self.src_party_id = src_party_id - self.role = role - self.party_id = party_id - self.initiator = initiator + def __init__(self, initiator_party_id, roles, component_list, dataset_list, dag_schema, component_parameters): + self.party_id = initiator_party_id self.roles = roles self.component_list = component_list self.dataset_list = dataset_list - self.run_time_conf = runtime_conf - self.dsl = dsl + self.dag_schema = dag_schema self.component_parameters = component_parameters class PermissionReturn(ParametersBase): - def __init__(self, code=RetCode.SUCCESS, message="success"): + def __init__(self, code=ReturnCode.Base.SUCCESS, message="success"): self.code = code self.message = message diff --git a/python/fate_flow/hook/flow/__init__.py b/python/fate_flow/hook/flow/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/hook/flow/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/hook/flow/client_authentication.py b/python/fate_flow/hook/flow/client_authentication.py index ad63f81e5..58f9e4b8d 100644 --- a/python/fate_flow/hook/flow/client_authentication.py +++ b/python/fate_flow/hook/flow/client_authentication.py @@ -1,63 +1,34 @@ -from base64 import b64encode -from hmac import HMAC -from time import time -from urllib.parse import quote, urlencode - -from fate_flow.settings import HTTP_APP_KEY, HTTP_SECRET_KEY, MAX_TIMESTAMP_INTERVAL -from fate_flow.entity import RetCode +from fate_flow.controller.permission import Authentication, PermissionController +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import InvalidParameter from fate_flow.hook import HookManager -from fate_flow.hook.common.parameters import ClientAuthenticationReturn, ClientAuthenticationParameters +from fate_flow.hook.common.parameters import AuthenticationReturn, AuthenticationParameters @HookManager.register_client_authentication_hook -def authentication(parm: ClientAuthenticationParameters) -> ClientAuthenticationReturn: - if not (HTTP_APP_KEY and HTTP_SECRET_KEY): - return ClientAuthenticationReturn(code=RetCode.AUTHENTICATION_ERROR, - message=f"settings HTTP_APP_KEY and HTTP_SECRET_KEY is None") - - requirement_parm = ['TIMESTAMP', 'NONCE', 'APP_KEY', 'SIGNATURE'] - for _p in requirement_parm: - if not parm.headers.get(_p): - return ClientAuthenticationReturn(code=RetCode.AUTHENTICATION_ERROR, - message=f"requirement headers parameters: {requirement_parm}") - try: - timestamp = int(parm.headers['TIMESTAMP']) / 1000 - except Exception: - return ClientAuthenticationReturn(code=RetCode.AUTHENTICATION_ERROR, message="Invalid TIMESTAMP") - now = time() - if not now - MAX_TIMESTAMP_INTERVAL < timestamp < now + MAX_TIMESTAMP_INTERVAL: - return ClientAuthenticationReturn( - code=RetCode.AUTHENTICATION_ERROR, - message=f'TIMESTAMP is more than {MAX_TIMESTAMP_INTERVAL} seconds away from the server time' - ) - - if not parm.headers['NONCE']: - return ClientAuthenticationReturn( - code=RetCode.AUTHENTICATION_ERROR, - message='Invalid NONCE' - ) +def authentication(parm: AuthenticationParameters) -> AuthenticationReturn: + app_id = parm.headers.get("appId") + user_name = parm.headers.get("userName") + timestamp = parm.headers.get("Timestamp") + nonce = parm.headers.get("Nonce") + signature = parm.headers.get("Signature") + check_parameters(app_id, user_name, timestamp, nonce, signature) + if Authentication.md5_verify(app_id, timestamp, nonce, signature, user_name): + if PermissionController.enforcer(app_id, parm.path, parm.method): + return AuthenticationReturn(code=ReturnCode.Base.SUCCESS, message="success") + else: + return AuthenticationReturn(code=ReturnCode.API.AUTHENTICATION_FAILED, + message="Authentication Failed") + else: + return AuthenticationReturn(code=ReturnCode.API.VERIFY_FAILED, message="varify failed!") - if parm.headers['APP_KEY'] != HTTP_APP_KEY: - return ClientAuthenticationReturn( - code=RetCode.AUTHENTICATION_ERROR, - message='Unknown APP_KEY' - ) - # authentication - signature = b64encode(HMAC(HTTP_SECRET_KEY.encode('ascii'), b'\n'.join([ - parm.headers['TIMESTAMP'].encode('ascii'), - parm.headers['NONCE'].encode('ascii'), - parm.headers['APP_KEY'].encode('ascii'), - parm.full_path.rstrip('?').encode('ascii'), - parm.data if parm.json else b'', - # quote_via: `urllib.parse.quote` replaces spaces with `%20` - # safe: unreserved characters from rfc3986 - urlencode(sorted(parm.form.items()), quote_via=quote, safe='-._~').encode('ascii') - if parm.form else b'', - ]), 'sha1').digest()).decode('ascii') - if signature != parm.headers['SIGNATURE']: - return ClientAuthenticationReturn( - code=RetCode.AUTHENTICATION_ERROR, - message='signature authentication failed' - ) - return ClientAuthenticationReturn() \ No newline at end of file +def check_parameters(app_id, user_name, time_stamp, nonce, signature): + if not app_id: + raise InvalidParameter(name="appId") + if not time_stamp or not isinstance(time_stamp, str): + raise InvalidParameter(name="Timestamp") + if not nonce or not isinstance(time_stamp, str) or len(nonce) != 4: + raise InvalidParameter(name="Nonce") + if not signature: + raise InvalidParameter(name="Signature") diff --git a/python/fate_flow/hook/flow/permission.py b/python/fate_flow/hook/flow/permission.py index 4d9237315..efb92951c 100644 --- a/python/fate_flow/hook/flow/permission.py +++ b/python/fate_flow/hook/flow/permission.py @@ -1,27 +1,22 @@ -from fate_flow.controller.permission_controller import PermissionCheck -from fate_flow.entity import RetCode +from fate_flow.controller.permission import PermissionCheck +from fate_flow.entity.code import ReturnCode from fate_flow.hook import HookManager from fate_flow.hook.common.parameters import PermissionCheckParameters, PermissionReturn -from fate_flow.settings import COMPONENT_PERMISSION, DATASET_PERMISSION +from fate_flow.runtime.system_settings import LOCAL_PARTY_ID, PARTY_ID @HookManager.register_permission_check_hook def permission(parm: PermissionCheckParameters) -> PermissionReturn: - if parm.role == "local" or str(parm.party_id) == "0": - return PermissionReturn() - - if parm.src_party_id == parm.party_id: + if parm.party_id == LOCAL_PARTY_ID or parm.party_id == PARTY_ID: return PermissionReturn() checker = PermissionCheck(**parm.to_dict()) + component_result = checker.check_component() - if COMPONENT_PERMISSION: - component_result = checker.check_component() - if component_result.code != RetCode.SUCCESS: - return component_result + if component_result.code != ReturnCode.Base.SUCCESS: + return component_result - if DATASET_PERMISSION: - dataset_result = checker.check_dataset() - if dataset_result.code != RetCode.SUCCESS: - return dataset_result + dataset_result = checker.check_dataset() + if dataset_result.code != ReturnCode.Base.SUCCESS: + return dataset_result return PermissionReturn() diff --git a/python/fate_flow/hook/flow/site_authentication.py b/python/fate_flow/hook/flow/site_authentication.py index c87795303..12c132f92 100644 --- a/python/fate_flow/hook/flow/site_authentication.py +++ b/python/fate_flow/hook/flow/site_authentication.py @@ -1,36 +1,66 @@ -import base64 -import json +import hashlib -from Crypto.PublicKey import RSA -from Crypto.Signature import PKCS1_v1_5 -from Crypto.Hash import SHA256 - -from fate_flow.db.key_manager import RsaKeyManager -from fate_flow.entity import RetCode -from fate_flow.entity.types import SiteKeyName +from fate_flow.controller.permission import PermissionController, Authentication +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import NoFoundAppid from fate_flow.hook import HookManager -from fate_flow.hook.common.parameters import SignatureParameters, AuthenticationParameters, AuthenticationReturn, \ - SignatureReturn -from fate_flow.settings import PARTY_ID +from fate_flow.hook.common.parameters import SignatureParameters, SignatureReturn, AuthenticationParameters, \ + AuthenticationReturn +from fate_flow.manager.service.app_manager import AppManager +from fate_flow.runtime.system_settings import LOCAL_PARTY_ID, PARTY_ID @HookManager.register_site_signature_hook def signature(parm: SignatureParameters) -> SignatureReturn: - private_key = RsaKeyManager.get_key(parm.party_id, key_name=SiteKeyName.PRIVATE.value) - if not private_key: - raise Exception(f"signature error: no found party id {parm.party_id} private key") - sign = PKCS1_v1_5.new(RSA.importKey(private_key)).sign(SHA256.new(json.dumps(parm.body).encode())) - return SignatureReturn(site_signature=base64.b64encode(sign).decode()) + if parm.party_id == LOCAL_PARTY_ID: + parm.party_id = PARTY_ID + apps = AppManager.query_partner_app(party_id=parm.party_id) + if not apps: + e = NoFoundAppid(party_id=parm.party_id) + return SignatureReturn( + code=e.code, + message=e.message + ) + app = apps[0] + nonce = Authentication.generate_nonce() + timestamp = Authentication.generate_timestamp() + initiator_party_id = parm.initiator_party_id if parm.initiator_party_id else "" + key = hashlib.md5(str(app.f_app_id + initiator_party_id + nonce + timestamp).encode("utf8")).hexdigest().lower() + sign = hashlib.md5(str(key + app.f_app_token).encode("utf8")).hexdigest().lower() + + return SignatureReturn(signature={ + "Signature": sign, + "appId": app.f_app_id, + "Nonce": nonce, + "Timestamp": timestamp, + "initiatorPartyId": initiator_party_id + }) @HookManager.register_site_authentication_hook def authentication(parm: AuthenticationParameters) -> AuthenticationReturn: - party_id = parm.src_party_id if parm.src_party_id and str(parm.src_party_id) != "0" else PARTY_ID - public_key = RsaKeyManager.get_key(party_id=party_id, key_name=SiteKeyName.PUBLIC.value) - if not public_key: - raise Exception(f"signature error: no found party id {party_id} public key") - verifier = PKCS1_v1_5.new(RSA.importKey(public_key)) - if verifier.verify(SHA256.new(json.dumps(parm.body).encode()), base64.b64decode(parm.site_signature)) is True: - return AuthenticationReturn() + app_id = parm.headers.get("appId") + timestamp = parm.headers.get("Timestamp") + nonce = parm.headers.get("Nonce") + sign = parm.headers.get("Signature") + initiator_party_id = parm.headers.get("initiatorPartyId") + check_parameters(app_id, timestamp, nonce, sign) + if Authentication.md5_verify(app_id, timestamp, nonce, sign, initiator_party_id): + if PermissionController.enforcer(app_id, parm.path, parm.method): + return AuthenticationReturn(code=ReturnCode.Base.SUCCESS, message="success") + else: + return AuthenticationReturn(code=ReturnCode.API.AUTHENTICATION_FAILED, + message=f"Authentication Failed: app_id[{app_id}, path[{parm.path}, method[{parm.method}]]]") else: - return AuthenticationReturn(code=RetCode.AUTHENTICATION_ERROR, message="authentication failed") + return AuthenticationReturn(code=ReturnCode.API.VERIFY_FAILED, message="varify failed!") + + +def check_parameters(app_id, time_stamp, nonce, sign): + if not app_id: + raise ValueError(ReturnCode.API.INVALID_PARAMETER, "invalid parameter: appId") + if not time_stamp or not isinstance(time_stamp, str): + raise ValueError(ReturnCode.API.INVALID_PARAMETER, "invalid parameter:timeStamp") + if not nonce or not isinstance(time_stamp, str) or len(nonce) != 4: + raise ValueError(ReturnCode.API.INVALID_PARAMETER, "invalid parameter: Nonce") + if not sign: + raise ValueError(ReturnCode.API.INVALID_PARAMETER, "invalid parameter: Signature") diff --git a/python/fate_flow/hub/__init__.py b/python/fate_flow/hub/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/hub/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/hub/components_wraps/__init__.py b/python/fate_flow/hub/components_wraps/__init__.py new file mode 100644 index 000000000..c9fac8fd6 --- /dev/null +++ b/python/fate_flow/hub/components_wraps/__init__.py @@ -0,0 +1,26 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import abc +from abc import ABCMeta + + +class WrapsABC(metaclass=ABCMeta): + @abc.abstractmethod + def run(self): + ... + + @abc.abstractmethod + def cleanup(self): + ... diff --git a/python/fate_flow/external/storage/__init__.py b/python/fate_flow/hub/components_wraps/fate/__init__.py similarity index 87% rename from python/fate_flow/external/storage/__init__.py rename to python/fate_flow/hub/components_wraps/fate/__init__.py index 7897a23d0..8d9cac80f 100644 --- a/python/fate_flow/external/storage/__init__.py +++ b/python/fate_flow/hub/components_wraps/fate/__init__.py @@ -12,8 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -from fate_flow.external.storage.mysql import MysqlStorage - -__all__ = ["MysqlStorage"] +from fate_flow.hub.components_wraps.fate._wraps import FlowWraps +__all__ = ["FlowWraps"] diff --git a/python/fate_flow/hub/components_wraps/fate/_wraps.py b/python/fate_flow/hub/components_wraps/fate/_wraps.py new file mode 100644 index 000000000..9c8e2c3a1 --- /dev/null +++ b/python/fate_flow/hub/components_wraps/fate/_wraps.py @@ -0,0 +1,649 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io +import json +import logging +import os.path +import sys +import tarfile +import traceback +from typing import List + +import yaml + +from fate_flow.engine.backend import build_backend +from fate_flow.engine.storage import StorageEngine, DataType, Session +from fate_flow.entity.code import ReturnCode +from fate_flow.entity.spec.dag import PreTaskConfigSpec, DataWarehouseChannelSpec, ComponentIOArtifactsTypeSpec, \ + TaskConfigSpec, ArtifactInputApplySpec, Metadata, RuntimeTaskOutputChannelSpec, \ + ArtifactOutputApplySpec, ModelWarehouseChannelSpec, ArtifactOutputSpec, ComponentOutputMeta, TaskCleanupConfigSpec, \ + PartySpec + +from fate_flow.entity.types import DataframeArtifactType, TableArtifactType, TaskStatus, ComputingEngine, \ + JsonModelArtifactType, LauncherType + +from fate_flow.hub.components_wraps import WrapsABC +from fate_flow.manager.outputs.data import DataManager, DatasetManager +from fate_flow.runtime.system_settings import STANDALONE_DATA_HOME, DEFAULT_OUTPUT_DATA_PARTITIONS, \ + DEEPSPEED_MODEL_DIR_PLACEHOLDER, DEEPSPEED_LOGS_DIR_PLACEHOLDER +from fate_flow.utils import job_utils +from fate_flow.utils.job_utils import generate_deepspeed_id + +logger = logging.getLogger(__name__) + + +class FlowWraps(WrapsABC): + def __init__(self, config: PreTaskConfigSpec): + self.config = config + self.mlmd = self.load_mlmd(config.mlmd) + self.backend = build_backend(backend_name=self.config.conf.computing.type, launcher_name=self.config.launcher_name) + self._component_define = None + self._destroy_temp_data = [] + + @property + def task_info(self): + return { + "component": self.config.component, + "job_id": self.config.job_id, + "role": self.config.role, + "party_id": self.config.party_id, + "task_name": self.config.task_name, + "task_id": self.config.task_id, + "task_version": self.config.task_version + } + + @property + def task_input_dir(self): + return job_utils.get_task_directory(**self.task_info, input=True) + + @property + def task_output_dir(self): + return job_utils.get_task_directory(**self.task_info, output=True) + + def run(self): + code = 0 + exceptions = "" + try: + config = self.preprocess() + output_meta = self.run_component(config) + self.push_output(output_meta) + code = output_meta.status.code + exceptions = None + if output_meta.status.code != ReturnCode.Base.SUCCESS: + code = ReturnCode.Task.COMPONENT_RUN_FAILED + exceptions = output_meta.status.exceptions + logger.error(exceptions) + except Exception as e: + traceback.format_exc() + code = ReturnCode.Task.TASK_RUN_FAILED + exceptions = str(e) + logger.error(e) + finally: + self.destroy(code) + self.report_status(code, exceptions) + if code: + sys.exit(code) + + def cleanup(self): + config = TaskCleanupConfigSpec( + computing=self.config.conf.computing, + federation=self.config.conf.federation + ) + return self.backend.cleanup( + provider_name=self.config.provider_name, + config=config.dict(), + task_info=self.task_info, + party_task_id=self.config.party_task_id + ) + + def preprocess(self): + # input + logger.info("start generating input artifacts") + logger.info(self.config.input_artifacts) + input_artifacts = self._preprocess_input_artifacts() + logger.info("input artifacts are ready") + logger.debug(input_artifacts) + logger.info(f"PYTHON PATH: {os.environ.get('PYTHONPATH')}") + + # output + logger.info("start generating output artifacts") + output_artifacts = self._preprocess_output_artifacts() + logger.info(f"output_artifacts: {output_artifacts}") + logger_config = json.dumps(self.config.conf.logger.config) + if self.config.launcher_name == LauncherType.DEEPSPEED: + logger_config = logger_config.replace( + job_utils.get_job_log_directory(self.config.job_id), + os.path.join(DEEPSPEED_LOGS_DIR_PLACEHOLDER, self.config.job_id) + ) + self.config.conf.logger.config = json.loads(logger_config) + config = TaskConfigSpec( + job_id=self.config.job_id, + task_id=self.config.task_id, + party_task_id=self.config.party_task_id, + component=self.config.component, + role=self.config.role, + party_id=self.config.party_id, + stage=self.config.stage, + parameters=self.config.parameters, + input_artifacts=input_artifacts, + output_artifacts=output_artifacts, + conf=self.config.conf, + task_name=self.config.task_name + ) + logger.debug(config) + return config + + def run_component(self, config): + self._set_env() + task_parameters = config.dict() + logger.info("start run task") + os.makedirs(self.task_input_dir, exist_ok=True) + os.makedirs(self.task_output_dir, exist_ok=True) + conf_path = os.path.join(self.task_input_dir, "task_parameters.yaml") + task_result = os.path.join(self.task_output_dir, "task_result.yaml") + with open(conf_path, "w") as f: + yaml.dump(task_parameters, f) + code = self.backend.run( + provider_name=self.config.provider_name, + task_info=self.task_info, + engine_run=self.config.engine_run, + run_parameters=task_parameters, + output_path=task_result, + conf_path=conf_path, + session_id=generate_deepspeed_id(self.config.party_task_id), + sync=True + ) + logger.info(f"finish task, return code {code}") + if os.path.exists(task_result): + with open(task_result, "r") as f: + try: + result = json.load(f) + output_meta = ComponentOutputMeta.parse_obj(result) + if code != 0: + output_meta.status.code = code + # logger.debug(output_meta) + except Exception as e: + raise RuntimeError(f"Task run failed {e}, you can see the task result file for details: {task_result}.") + else: + output_meta = ComponentOutputMeta(status=ComponentOutputMeta.Status( + code=ReturnCode.Task.NO_FOUND_RUN_RESULT, + exceptions=f"No found task output." + )) + return output_meta + + def push_output(self, output_meta: ComponentOutputMeta): + if self.task_end_with_success(output_meta.status.code): + # push output data to server + if not output_meta.io_meta: + logger.info("No found io meta, pass push") + return + for key, datas in output_meta.io_meta.outputs.data.items(): + if isinstance(datas, list): + self._push_data(key, [ArtifactOutputSpec(**data) for data in datas]) + else: + self._push_data(key, [ArtifactOutputSpec(**datas)]) + + # push model + for key, models in output_meta.io_meta.outputs.model.items(): + if isinstance(models, list): + self._push_model(key, [ArtifactOutputSpec(**model) for model in models]) + else: + self._push_model(key, [ArtifactOutputSpec(**models)]) + + # push metric + for key, metrics in output_meta.io_meta.outputs.metric.items(): + if isinstance(metrics, list): + for metric in metrics: + output_metric = ArtifactOutputSpec(**metric) + self._push_metric(key, output_metric) + else: + output_metric = ArtifactOutputSpec(**metrics) + self._push_metric(key, output_metric) + + def _push_data(self, output_key, output_datas: List[ArtifactOutputSpec]): + logger.info("save data") + # logger.debug(f"key[{output_key}] output_datas[{output_datas}]") + for index, output_data in enumerate(output_datas): + if output_data.consumed is False: + # filter invalid output data + continue + namespace = output_data.metadata.namespace + name = output_data.metadata.name + if not namespace and not name: + namespace, name = DatasetManager.get_output_name(output_data.uri) + logger.info(f"save data tracking to {namespace}, {name}") + overview = output_data.metadata.data_overview + source = output_data.metadata.source + uri = output_data.uri + if output_data.type_name == DataType.DATA_UNRESOLVED: + uri = "" + # check namespace and name（reader） + resp = self.mlmd.query_data_meta(name=name, namespace=namespace) + if resp.json().get("code") != 0: + raise ValueError(f"Check failed[{resp.text}]") + resp = self.mlmd.save_data_tracking( + execution_id=self.config.party_task_id, + output_key=output_key, + meta_data=output_data.metadata.metadata.get("schema", {}), + uri=uri, + namespace=namespace, + name=name, + overview=overview.dict() if overview else {}, + source=source.dict() if source else {}, + data_type=output_data.type_name, + index=index, + partitions=DEFAULT_OUTPUT_DATA_PARTITIONS + ) + self.log_response(resp, req_info="save data tracking") + + def _push_model(self, output_key, output_models: List[ArtifactOutputSpec]): + logger.info("save model") + logger.info(f"key[{output_key}] output_models[{output_models}]") + tar_io = io.BytesIO() + if self.config.launcher_name == LauncherType.DEEPSPEED: + logger.info("pass") + return + for output_model in output_models: + engine, address = DataManager.uri_to_address(output_model.uri) + if engine == StorageEngine.FILE: + _path = address.path + if os.path.exists(_path): + if os.path.isdir(_path): + path = _path + else: + path = os.path.dirname(_path) + model_key = os.path.basename(_path) + meta_path = os.path.join(path, f"{model_key}.meta.yaml") + with open(meta_path, "w") as fp: + output_model.metadata.model_key = model_key + output_model.metadata.index = output_model.metadata.source.output_index + output_model.metadata.type_name = output_model.type_name + yaml.dump(output_model.metadata.dict(), fp) + # tar and send to server + tar_io = self._tar_model(tar_io=tar_io, path=path) + type_name = output_model.type_name + else: + logger.warning(f"No found model path: {_path}") + else: + raise ValueError(f"Engine {engine} is not supported") + if output_models: + resp = self.mlmd.save_model( + model_id=self.config.model_id, + model_version=self.config.model_version, + execution_id=self.config.party_task_id, + output_key=output_key, + fp=tar_io, + type_name=type_name + ) + self.log_response(resp, req_info="save model") + + @staticmethod + def no_metadata_filter(tarinfo): + tarinfo.pax_headers = {} + return tarinfo + + @classmethod + def _tar_model(cls, tar_io, path): + with tarfile.open(fileobj=tar_io, mode="x:tar") as tar: + for _root, _dir, _files in os.walk(path): + for _f in _files: + full_path = os.path.join(_root, _f) + rel_path = os.path.relpath(full_path, path) + tar.add(full_path, rel_path, filter=cls.no_metadata_filter) + tar_io.seek(0) + return tar_io + + def _push_metric(self, output_key, output_metric: ArtifactOutputSpec): + logger.info(f"output metric: {output_metric}") + logger.info("save metric") + engine, address = DataManager.uri_to_address(output_metric.uri) + if engine == StorageEngine.FILE: + _path = address.path + if os.path.exists(_path): + with open(_path, "r") as f: + data = json.load(f) + if data: + resp = self.mlmd.save_metric( + execution_id=self.config.party_task_id, + data=data + ) + self.log_response(resp, req_info="save metric") + else: + logger.warning(f"No found metric path: {_path}") + else: + pass + + @staticmethod + def log_response(resp, req_info): + try: + logger.info(resp.json()) + resp_json = resp.json() + if resp_json.get("code") != ReturnCode.Base.SUCCESS: + logging.exception(f"{req_info}: {resp.text}") + except Exception: + logger.error(f"{req_info}: {resp.text}") + + def _preprocess_input_artifacts(self): + input_artifacts = {} + if self.config.input_artifacts.data: + for _k, _channels in self.config.input_artifacts.data.items(): + if isinstance(_channels, list): + input_artifacts[_k] = [] + for _channel in _channels: + _artifacts = self._intput_data_artifacts(_k, _channel) + if _artifacts: + input_artifacts[_k].append(_artifacts) + elif self._check_is_multi_input_data(_k): + input_artifacts[_k] = [self._intput_data_artifacts(_k, _channels)] + else: + input_artifacts[_k] = self._intput_data_artifacts(_k, _channels) + if not input_artifacts[_k]: + input_artifacts.pop(_k) + + if self.config.input_artifacts.model: + for _k, _channels in self.config.input_artifacts.model.items(): + if isinstance(_channels, list): + input_artifacts[_k] = [] + for _channel in _channels: + input_artifacts[_k].append(self._intput_model_artifacts(_k, _channel)) + elif self._check_is_multi_input_model(_k): + input_artifacts[_k] = [self._intput_model_artifacts(_k, _channels)] + else: + input_artifacts[_k] = self._intput_model_artifacts(_k, _channels) + if not input_artifacts[_k]: + input_artifacts.pop(_k) + return input_artifacts + + def _check_is_multi_input_model(self, key): + for define in self.component_define.inputs.model: + if define.name == key and define.is_multi: + return True + return False + + def _check_is_multi_input_data(self, key): + for define in self.component_define.inputs.data: + if define.name == key and define.is_multi: + return True + return False + + def _preprocess_output_artifacts(self): + # get component define + logger.debug("get component define") + define = self.component_define + logger.info(f"component define: {define}") + output_artifacts = {} + if not define: + return output_artifacts + else: + # data + for key in define.outputs.dict().keys(): + datas = getattr(define.outputs, key, None) + if datas: + for data in datas: + _output_artifacts = [] + for data_type in data.types: + _output_artifacts.append(self._output_artifacts(data_type.type_name, data.is_multi, + data.name, key)) + output_artifacts[data.name] = _output_artifacts[0] + return output_artifacts + + def _set_env(self): + if self.config.conf.computing.type == ComputingEngine.STANDALONE or \ + self.config.conf.federation.type == ComputingEngine.STANDALONE: + os.environ["STANDALONE_DATA_PATH"] = STANDALONE_DATA_HOME + + def _output_artifacts(self, type_name, is_multi, name, output_type=None): + output_artifacts = ArtifactOutputApplySpec(uri="", type_name=type_name) + if type_name in [DataframeArtifactType.type_name, TableArtifactType.type_name]: + uri = DatasetManager.output_data_uri(self.config.conf.storage, self.config.task_id, is_multi=is_multi) + else: + if output_type == "metric": + # api path + uri = self.mlmd.get_metric_save_url(execution_id=self.config.party_task_id) + else: + base_dir = "" + if self.config.launcher_name == LauncherType.DEEPSPEED: + base_dir = DEEPSPEED_MODEL_DIR_PLACEHOLDER + uri = DatasetManager.output_local_uri( + task_info=self.task_info, name=name, type_name=type_name, is_multi=is_multi, + base_dir=base_dir + ) + output_artifacts.uri = uri + return output_artifacts + + @property + def component_define(self) -> ComponentIOArtifactsTypeSpec: + if not self._component_define: + self.set_component_define() + return self._component_define + + def set_component_define(self): + define = self.backend.get_component_define( + provider_name=self.config.provider_name, + task_info=self.task_info, + stage=self.config.stage + ) + if define: + self._component_define = ComponentIOArtifactsTypeSpec(**define) + + def _intput_data_artifacts(self, key, channel): + if not job_utils.check_party_in(self.config.role, self.config.party_id, channel.parties): + logger.info(f"role {self.config.role} does not require intput data artifacts") + return + # data reference conversion + meta = ArtifactInputApplySpec( + metadata=Metadata( + metadata=dict(options=dict(partitions=self.config.computing_partitions)) + ), + uri="" + ) + query_field = {} + logger.info(f"get key[{key}] channel[{channel}]") + if isinstance(channel, DataWarehouseChannelSpec): + # external data reference -> data meta + if channel.name and channel.namespace: + query_field = { + "namespace": channel.namespace, + "name": channel.name + } + else: + query_field = { + "job_id": channel.job_id, + "role": self.config.role, + "party_id": self.config.party_id, + "task_name": channel.producer_task, + "output_key": channel.output_artifact_key + } + + elif isinstance(channel, RuntimeTaskOutputChannelSpec): + # this job output data reference -> data meta + query_field = { + "job_id": self.config.job_id, + "role": self.config.role, + "party_id": self.config.party_id, + "task_name": channel.producer_task, + "output_key": channel.output_artifact_key + } + logger.info(f"query data: [{query_field}]") + resp = self.mlmd.query_data_meta(**query_field) + logger.debug(resp.text) + resp_json = resp.json() + if resp_json.get("code") != 0: + # Judging whether to optional + for input_data_define in self.component_define.inputs.data: + if input_data_define.name == key and input_data_define.optional: + logger.info(f"component define input data name {key} optional {input_data_define.optional}") + return + raise ValueError(f"Get data artifacts failed: {query_field}, response: {resp.text}") + resp_data = resp_json.get("data", []) + logger.info(f"intput data artifacts are ready") + if len(resp_data) == 1: + data = resp_data[0] + schema = data.get("meta", {}) + meta.metadata.metadata.update({"schema": schema}) + meta.type_name = data.get("data_type") + if meta.type_name == DataType.TABLE: + # destroy table data + self._destroy_temp_data.append((data.get("namespace"), data.get("name"))) + meta.uri = data.get("path") + source = data.get("source", {}) + if source: + meta.metadata.source = source + return meta + elif len(resp_data) > 1: + meta_list = [] + for data in resp_data: + schema = data.get("meta", {}) + meta.metadata.metadata.update({"schema": schema}) + meta.uri = data.get("path") + meta.type_name = data.get("data_type") + source = data.get("source", {}) + if source: + meta.metadata.source = source + meta_list.append(meta) + return meta_list + else: + raise RuntimeError(resp_data) + + def _intput_model_artifacts(self, key, channel): + if not job_utils.check_party_in(self.config.role, self.config.party_id, channel.parties): + logger.info(f"role {self.config.role} does not require intput model artifacts") + return + # model reference conversion + meta = ArtifactInputApplySpec(metadata=Metadata(metadata={}), uri="") + query_field = { + "task_name": channel.producer_task, + "output_key": channel.output_artifact_key, + "role": self.config.role, + "party_id": self.config.party_id + } + logger.info(f"get key[{key}] channel[{channel}]") + if isinstance(channel, ModelWarehouseChannelSpec): + # external model reference -> download to local + if channel.model_id and channel.model_version: + query_field.update({ + "model_id": channel.model_id, + "model_version": channel.model_version + }) + else: + query_field.update({ + "model_id": self.config.model_id, + "model_version": self.config.model_version + }) + elif isinstance(channel, RuntimeTaskOutputChannelSpec): + query_field.update({ + "model_id": self.config.model_id, + "model_version": self.config.model_version + }) + + logger.info(f"query model: [{query_field}]") + + # this job output data reference -> data meta + input_model_base = os.path.join(self.task_input_dir, "model") + os.makedirs(input_model_base, exist_ok=True) + _io = io.BytesIO() + resp = self.mlmd.download_model(**query_field) + if resp.headers.get('content-type') == 'application/json': + raise RuntimeError(f"Download model failed, {resp.text}") + try: + for chunk in resp.iter_content(1024): + if chunk: + _io.write(chunk) + _io.seek(0) + model = tarfile.open(fileobj=_io) + except Exception as e: + for input_data_define in self.component_define.inputs.model: + if input_data_define.name == key and input_data_define.optional: + logger.info(f"component define input model name {key} optional {input_data_define.optional}") + return + raise RuntimeError(f"Download model failed: {query_field}") + logger.info(f"intput model artifacts are ready: {model.getnames()}") + metas = [] + file_names = model.getnames() + for name in file_names: + if name.endswith("yaml"): + fp = model.extractfile(name).read() + model_meta = yaml.safe_load(fp) + model_meta = Metadata.parse_obj(model_meta) + model_task_id = model_meta.source.task_id if model_meta.source.task_id else "" + input_model_file = os.path.join(input_model_base, "_".join([model_task_id, model_meta.model_key])) + if model_meta.type_name not in [JsonModelArtifactType.type_name]: + self._write_model_dir(model, input_model_file) + else: + model_fp = model.extractfile(model_meta.model_key).read() + with open(input_model_file, "wb") as fw: + fw.write(model_fp) + meta.uri = f"file://{input_model_file}" + meta.metadata = model_meta + metas.append(meta) + if not metas: + raise RuntimeError(f"Download model failed: {query_field}") + if len(metas) == 1: + return metas[0] + return metas + + @staticmethod + def _write_model_dir(model, path): + for name in model.getnames(): + if not name.endswith("yaml"): + model_fp = model.extractfile(name).read() + input_model_file = os.path.join(path, name) + os.makedirs(os.path.dirname(input_model_file), exist_ok=True) + with open(input_model_file, "wb") as fw: + fw.write(model_fp) + + def report_status(self, code, error=""): + if self.task_end_with_success(code): + resp = self.mlmd.report_task_status( + execution_id=self.config.party_task_id, + status=TaskStatus.SUCCESS + ) + else: + resp = self.mlmd.report_task_status( + execution_id=self.config.party_task_id, + status=TaskStatus.FAILED, + error=error + ) + self.log_response(resp, req_info="report status") + + @staticmethod + def task_end_with_success(code): + return code == 0 + + def destroy(self, code): + if self.task_end_with_success(code): + for namespace, name in self._destroy_temp_data: + try: + logger.info(f"destroy table {namespace}, {name}") + with Session() as sess: + table = sess.get_table( + name=name, + namespace=namespace + ) + table.destroy() + logger.info(f"destroy table success") + except Exception as e: + logger.error(e) + + @staticmethod + def load_mlmd(mlmd): + if mlmd.type == "flow": + from ofx.api.client import FlowSchedulerApi + client = FlowSchedulerApi( + host=mlmd.metadata.get("host"), + port=mlmd.metadata.get("port"), + protocol=mlmd.metadata.get("protocol"), + api_version=mlmd.metadata.get("api_version")) + return client.worker diff --git a/python/fate_flow/hub/database/__init__.py b/python/fate_flow/hub/database/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/hub/database/mysql.py b/python/fate_flow/hub/database/mysql.py new file mode 100644 index 000000000..72bcc83cc --- /dev/null +++ b/python/fate_flow/hub/database/mysql.py @@ -0,0 +1,25 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from playhouse.pool import PooledMySQLDatabase + +from fate_flow.utils.password_utils import decrypt_database_config + + +def get_database_connection(config, decrypt_key): + database_config = config.copy() + db_name = database_config.pop("name") + decrypt_database_config(database_config, decrypt_key=decrypt_key) + return PooledMySQLDatabase(db_name, **database_config) diff --git a/python/fate_flow/CODE_STYLE.text b/python/fate_flow/hub/database/sqlite.py similarity index 62% rename from python/fate_flow/CODE_STYLE.text rename to python/fate_flow/hub/database/sqlite.py index 4cbde0b99..024c67eca 100644 --- a/python/fate_flow/CODE_STYLE.text +++ b/python/fate_flow/hub/database/sqlite.py @@ -13,12 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # -1. Based on Google Python Code Style -2. The log or message string begins with an lowercase letter -3. Strings use double quotes -4. Import Module but not directly import Class -5. Using {} instead of Dict -6. Using [] instead of List -7. Function parameters need to be typed -8. Function return need to be typed -8. Any expression is as clear and unambiguous as possible +from peewee import Insert + +from fate_flow.runtime.system_settings import SQLITE_PATH + + +def get_database_connection(config, decrypt_key): + Insert.on_conflict = lambda self, *args, **kwargs: self.on_conflict_replace() + from playhouse.apsw_ext import APSWDatabase + path = config.get("path") + if not path: + path = SQLITE_PATH + return APSWDatabase(path) diff --git a/python/fate_flow/hub/encrypt/__init__.py b/python/fate_flow/hub/encrypt/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/hub/encrypt/password_encrypt.py b/python/fate_flow/hub/encrypt/password_encrypt.py new file mode 100644 index 000000000..664ee43b2 --- /dev/null +++ b/python/fate_flow/hub/encrypt/password_encrypt.py @@ -0,0 +1,41 @@ +import base64 + +from Crypto import Random +from Crypto.PublicKey import RSA +from Crypto.Cipher import PKCS1_v1_5 as PKCS1_cipher + + +def rsa_key_generate(): + random_generator = Random.new().read + rsa = RSA.generate(2048, random_generator) + private_pem = rsa.exportKey().decode() + public_pem = rsa.publickey().exportKey().decode() + with open('private_key.pem', "w") as f: + f.write(private_pem) + with open('public_key.pem', "w") as f: + f.write(public_pem) + return private_pem, public_pem + + +def encrypt_data(public_key, msg): + cipher = PKCS1_cipher.new(RSA.importKey(public_key)) + encrypt_text = base64.b64encode(cipher.encrypt(bytes(msg.encode("utf8")))) + return encrypt_text.decode('utf-8') + + +def pwdecrypt(private_key, encrypt_msg): + try: + cipher = PKCS1_cipher.new(RSA.importKey(private_key)) + back_text = cipher.decrypt(base64.b64decode(encrypt_msg), 0) + return back_text.decode('utf-8') + except Exception as e: + raise RuntimeError(f"passwd decrypt failed: {e}") + + +def test_encrypt_decrypt(): + msg = "fate" + private_key, public_key = rsa_key_generate() + encrypt_text = encrypt_data(public_key, msg) + print(encrypt_text) + decrypt_text = pwdecrypt(private_key, encrypt_text) + print(msg == decrypt_text) diff --git a/python/fate_flow/hub/flow_hub.py b/python/fate_flow/hub/flow_hub.py new file mode 100644 index 000000000..c323d9e2b --- /dev/null +++ b/python/fate_flow/hub/flow_hub.py @@ -0,0 +1,57 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from importlib import import_module + +from fate_flow.entity.types import ProviderName, ProviderDevice +from fate_flow.runtime.component_provider import ComponentProvider +from fate_flow.runtime.system_settings import DEFAULT_COMPONENTS_WRAPS_MODULE + + +class FlowHub: + @staticmethod + def load_components_wraps(config, module_name=None): + if not module_name: + module_name = DEFAULT_COMPONENTS_WRAPS_MODULE + class_name = module_name.split(".")[-1] + module = ".".join(module_name.split(".")[:-1]) + return getattr(import_module(module), class_name)(config) + + @staticmethod + def load_provider_entrypoint(provider: ComponentProvider): + entrypoint = None + if provider.name == ProviderName.FATE and provider.device == ProviderDevice.LOCAL: + from fate_flow.hub.provider.local import LocalFateEntrypoint + entrypoint = LocalFateEntrypoint(provider) + elif provider.name == ProviderName.FATE_FLOW: + from fate_flow.hub.provider.local import FATEFLowEntrypoint + entrypoint = FATEFLowEntrypoint(provider) + elif provider.device == ProviderDevice.DOCKER: + from fate_flow.hub.provider.docker import DockerEntrypoint + entrypoint = DockerEntrypoint(provider) + return entrypoint + + @staticmethod + def load_database(engine_name, config, decrypt_key): + try: + return getattr(import_module(f"fate_flow.hub.database.{engine_name}"), "get_database_connection")( + config, decrypt_key) + except Exception as e: + try: + import_module(f"fate_flow.hub.database.{engine_name}") + except: + raise SystemError(f"Not support database engine {engine_name}") + raise SystemError(f"load engine {engine_name} function " + f"fate_flow.hub.database.{engine_name}.get_database_connection failed: {e}") diff --git a/python/fate_flow/hub/provider/__init__.py b/python/fate_flow/hub/provider/__init__.py new file mode 100644 index 000000000..fcb06a784 --- /dev/null +++ b/python/fate_flow/hub/provider/__init__.py @@ -0,0 +1,28 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import abc +from typing import List, Dict + + +class EntrypointABC: + @property + @abc.abstractmethod + def component_list(self) -> List: + ... + + @property + @abc.abstractmethod + def component_description(self) -> Dict: + ... diff --git a/python/fate_flow/hub/provider/docker.py b/python/fate_flow/hub/provider/docker.py new file mode 100644 index 000000000..9ff0cc597 --- /dev/null +++ b/python/fate_flow/hub/provider/docker.py @@ -0,0 +1,44 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re +from abc import ABC + +from fate_flow.hub.provider import EntrypointABC +from fate_flow.manager.container.docker_manager import DockerManager + + +class DockerEntrypoint(EntrypointABC, ABC): + def __init__(self, provider): + self.provider = provider + self.manager = DockerManager(provider) + + @property + def component_list(self): + return self.component_dict.keys() + + @property + def component_dict(self): + labels = self.manager.get_labels() + _dict = {} + pattern = r'^component\.\d*\.name$' + for key, cpn_name in labels.items(): + if re.match(pattern, key): + _k = key.rstrip(".name") + _dict[key] = cpn_name + return _dict + + @property + def component_description(self): + return {} diff --git a/python/fate_flow/hub/provider/local.py b/python/fate_flow/hub/provider/local.py new file mode 100644 index 000000000..a8538c9d8 --- /dev/null +++ b/python/fate_flow/hub/provider/local.py @@ -0,0 +1,51 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys + +from fate_flow.hub.provider import EntrypointABC + + +class LocalFateEntrypoint(EntrypointABC): + def __init__(self, provider): + self.provider = provider + + @property + def component_list(self): + if self.provider.python_path and self.provider.python_path not in sys.path: + sys.path.append(self.provider.python_path) + from fate.components.core import list_components + # {'buildin': [], 'thirdparty': []} + components = list_components() + _list = components.get('buildin', []) + _list.extend(components.get("thirdparty", [])) + return _list + + @property + def component_description(self): + return {} + + +class FATEFLowEntrypoint(EntrypointABC): + def __init__(self, provider): + self.provider = provider + + @property + def component_list(self): + from fate_flow.components.components import BUILDIN_COMPONENTS + return [component.name for component in BUILDIN_COMPONENTS] + + @property + def component_description(self): + return {} diff --git a/python/fate_flow/manager/__init__.py b/python/fate_flow/manager/__init__.py index 1eb6e4b2a..ae946a49c 100644 --- a/python/fate_flow/manager/__init__.py +++ b/python/fate_flow/manager/__init__.py @@ -12,5 +12,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# - diff --git a/python/fate_flow/manager/cache_manager.py b/python/fate_flow/manager/cache_manager.py deleted file mode 100644 index 49d09a605..000000000 --- a/python/fate_flow/manager/cache_manager.py +++ /dev/null @@ -1,96 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import typing -from uuid import uuid1 - -from fate_arch import session, storage -from fate_arch.abc import CTableABC -from fate_arch.common import DTable -from fate_arch.common.base_utils import current_timestamp -from fate_flow.db.db_models import DB, CacheRecord -from fate_flow.entity import DataCache - - -class CacheManager: - @classmethod - def persistent(cls, cache_name: str, cache_data: typing.Dict[str, CTableABC], cache_meta: dict, output_namespace: str, - output_name: str, output_storage_engine: str, output_storage_address: dict, - token=None) -> DataCache: - cache = DataCache(name=cache_name, meta=cache_meta) - for name, table in cache_data.items(): - table_meta = session.Session.persistent(computing_table=table, - namespace=output_namespace, - name=f"{output_name}_{name}", - schema=None, - engine=output_storage_engine, - engine_address=output_storage_address, - token=token) - cache.data[name] = DTable(namespace=table_meta.namespace, name=table_meta.name, - partitions=table_meta.partitions) - return cache - - @classmethod - def load(cls, cache: DataCache) -> typing.Tuple[typing.Dict[str, CTableABC], dict]: - cache_data = {} - for name, table in cache.data.items(): - storage_table_meta = storage.StorageTableMeta(name=table.name, namespace=table.namespace) - computing_table = session.get_computing_session().load( - storage_table_meta.get_address(), - schema=storage_table_meta.get_schema(), - partitions=table.partitions) - cache_data[name] = computing_table - return cache_data, cache.meta - - @classmethod - @DB.connection_context() - def record(cls, cache: DataCache, job_id: str = None, role: str = None, party_id: int = None, component_name: str = None, task_id: str = None, task_version: int = None, - cache_name: str = None): - for attr in {"job_id", "component_name", "task_id", "task_version"}: - if getattr(cache, attr) is None and locals().get(attr) is not None: - setattr(cache, attr, locals().get(attr)) - record = CacheRecord() - record.f_create_time = current_timestamp() - record.f_cache_key = uuid1().hex - cache.key = record.f_cache_key - record.f_cache = cache - record.f_job_id = job_id - record.f_role = role - record.f_party_id = party_id - record.f_component_name = component_name - record.f_task_id = task_id - record.f_task_version = task_version - record.f_cache_name = cache_name - rows = record.save(force_insert=True) - if rows != 1: - raise Exception("save cache tracking failed") - return record.f_cache_key - - @classmethod - @DB.connection_context() - def query(cls, cache_key: str = None, role: str = None, party_id: int = None, component_name: str = None, cache_name: str = None, - **kwargs) -> typing.List[DataCache]: - if cache_key is not None: - records = CacheRecord.query(cache_key=cache_key) - else: - records = CacheRecord.query(role=role, party_id=party_id, component_name=component_name, - cache_name=cache_name, **kwargs) - return [record.f_cache for record in records] - - @classmethod - @DB.connection_context() - def query_record(cls, role: str = None, party_id: int = None, component_name: str = None, **kwargs) -> typing.List[CacheRecord]: - records = CacheRecord.query(role=role, party_id=party_id, component_name=component_name, **kwargs) - return [record for record in records] diff --git a/python/fate_flow/protobuf/__init__.py b/python/fate_flow/manager/components/__init__.py similarity index 100% rename from python/fate_flow/protobuf/__init__.py rename to python/fate_flow/manager/components/__init__.py diff --git a/python/fate_flow/manager/components/base.py b/python/fate_flow/manager/components/base.py new file mode 100644 index 000000000..8be0b2fcb --- /dev/null +++ b/python/fate_flow/manager/components/base.py @@ -0,0 +1,49 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from fate_flow.entity.spec.dag import PartySpec, DAGSchema, DAGSpec, JobConfSpec, TaskConfSpec, TaskSpec, \ + PartyTaskSpec, PartyTaskRefSpec, RuntimeInputArtifacts +from fate_flow.manager.service.provider_manager import ProviderManager + + +class Base: + @staticmethod + def local_dag_schema(task_name, component_ref, parameters, inputs=None, provider=None, role=None, party_id=None): + if not provider: + provider = ProviderManager.get_fate_flow_provider() + if not role or not party_id: + role = "local" + party_id = "0" + party = PartySpec(role=role, party_id=[party_id]) + dag = DAGSchema( + schema_version=provider.version, + dag=DAGSpec( + conf=JobConfSpec(task=TaskConfSpec(provider=provider.provider_name)), + parties=[party], + stage="default", + tasks={task_name: TaskSpec( + component_ref=component_ref, + parties=[party], + conf=dict(provider=provider.provider_name) + )}, + party_tasks={ + f"{role}_{party_id}": PartyTaskSpec( + parties=[party], + tasks={task_name: PartyTaskRefSpec(parameters=parameters)} + )} + )) + if inputs: + dag.dag.tasks[task_name].inputs = RuntimeInputArtifacts(**inputs) + return dag diff --git a/python/fate_flow/manager/components/component_manager.py b/python/fate_flow/manager/components/component_manager.py new file mode 100644 index 000000000..f3cfb8bc1 --- /dev/null +++ b/python/fate_flow/manager/components/component_manager.py @@ -0,0 +1,158 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import uuid +import os +from tempfile import NamedTemporaryFile + + +from fate_flow.controller.job import JobController +from fate_flow.entity.code import ReturnCode +from fate_flow.entity.spec.dag import PartyTaskRefSpec, TaskSpec, PartySpec, RuntimeInputArtifacts, \ + RuntimeTaskOutputChannelSpec +from fate_flow.entity.types import EngineType +from fate_flow.manager.components.base import Base +from fate_flow.manager.service.provider_manager import ProviderManager +from fate_flow.runtime.system_settings import ENGINES, STORAGE, TEMP_DIR +from fate_flow.engine import storage +from fate_flow.errors.server_error import ExistsTable +from fate_flow.utils.file_utils import save_file + + +class ComponentManager(Base): + @classmethod + def upload(cls, file, head, partitions, meta, namespace, name, extend_sid, temp_path=None): + parameters = { + "file": file, + "head": head, + "partitions": partitions, + "meta": meta, + "extend_sid": extend_sid, + "is_temp_file": True if temp_path else False + } + if not name or not namespace: + name = str(uuid.uuid1()) + namespace = "upload" + parameters.update({ + "storage_engine": ENGINES.get(EngineType.STORAGE), + "name": name, + "namespace": namespace + }) + address = STORAGE.get(ENGINES.get(EngineType.STORAGE)) + if address: + parameters.update({"address": address}) + dag_schema = cls.local_dag_schema( + task_name="upload_0", + component_ref="upload", + parameters=parameters + ) + result = JobController.request_create_job(dag_schema, is_local=True) + if result.get("code") == ReturnCode.Base.SUCCESS: + result["data"] = {"name": name, "namespace": namespace} + return result + + @classmethod + def dataframe_transformer(cls, data_warehouse, namespace, name, drop, site_name): + data_table_meta = storage.StorageTableMeta(name=name, namespace=namespace) + if data_table_meta: + if not drop: + raise ExistsTable( + name=name, + namespace=namespace, + warning="If you want to ignore this error and continue transformer, " + "you can set the parameter of 'drop' to 'true' " + ) + data_table_meta.destroy_metas() + provider = ProviderManager.get_default_fate_provider() + dag_schema = cls.local_dag_schema( + task_name="transformer_0", + component_ref="dataframe_transformer", + parameters={"namespace": namespace, "name": name, "site_name": site_name}, + inputs={"data": {"table": {"data_warehouse": data_warehouse}}}, + provider=provider + ) + result = JobController.request_create_job(dag_schema, is_local=True) + if result.get("code") == ReturnCode.Base.SUCCESS: + result["data"] = {"name": name, "namespace": namespace} + return result + + @classmethod + def download(cls, namespace, name, path): + dag_schema = cls.local_dag_schema( + task_name="download_0", + component_ref="download", + parameters=dict(namespace=namespace, name=name, path=path) + ) + result = JobController.request_create_job(dag_schema, is_local=True) + if result.get("code") == ReturnCode.Base.SUCCESS: + result["data"] = {"name": name, "namespace": namespace, "path": path} + return result + + @classmethod + def upload_dataframe(cls, file, head, partitions, meta, namespace, name, extend_sid, is_temp_file=False): + parameters = { + "file": file, + "head": head, + "partitions": partitions, + "meta": meta, + "extend_sid": extend_sid, + "is_temp_file": is_temp_file + } + address = STORAGE.get(ENGINES.get(EngineType.STORAGE)) + if address: + parameters.update({"address": address}) + role = "local" + party_id = "0" + upload_name = "upload_0" + upload_ref = "upload" + transformer_name = "transformer_0" + transformer_ref = "dataframe_transformer" + + dag_schema = cls.local_dag_schema( + task_name=upload_name, + component_ref=upload_ref, + parameters=parameters, + role=role, + party_id=party_id + ) + dag_schema.dag.party_tasks[f"{role}_{party_id}"].tasks[transformer_name] = PartyTaskRefSpec( + parameters={"namespace": namespace, "name": name} + ) + + fate_provider = ProviderManager.get_default_fate_provider() + + dag_schema.dag.tasks[transformer_name] = TaskSpec( + component_ref=transformer_ref, + parties=[PartySpec(role=role, party_id=[party_id])], + conf=dict({"provider": fate_provider.provider_name}), + inputs=RuntimeInputArtifacts( + data={ + "table": { + "task_output_artifact": + RuntimeTaskOutputChannelSpec(producer_task=upload_name, output_artifact_key="table")} + }) + ) + result = JobController.request_create_job(dag_schema, is_local=True) + if result.get("code") == ReturnCode.Base.SUCCESS: + result["data"] = {"name": name, "namespace": namespace} + return result + + @classmethod + def upload_file(cls, file, head, partitions, meta, namespace, name, extend_sid): + os.makedirs(TEMP_DIR, exist_ok=True) + with NamedTemporaryFile(dir=TEMP_DIR, delete=False) as temp_file: + temp_path = temp_file.name + save_file(file, temp_path) + return cls.upload_dataframe(temp_path, head, partitions, meta, namespace, name, extend_sid, is_temp_file=True) diff --git a/python/fate_flow/manager/components/download.py b/python/fate_flow/manager/components/download.py new file mode 100644 index 000000000..6ba4a83ca --- /dev/null +++ b/python/fate_flow/manager/components/download.py @@ -0,0 +1,58 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +from fate_flow.engine import storage +from fate_flow.manager.outputs.data import DataManager + + +class Param(object): + def to_dict(self): + d = {} + for k, v in self.__dict__.items(): + if v is None: + continue + d[k] = v + return d + + +class DownloadParam(Param): + def __init__( + self, + dir_name="", + namespace="", + name="" + ): + self.dir_name = dir_name + self.namespace = namespace + self.name = name + + +class Download: + def __init__(self): + self.table = None + self.schema = {} + + def run(self, parameters: DownloadParam, job_id=""): + data_table_meta = storage.StorageTableMeta( + name=parameters.name, + namespace=parameters.namespace + ) + DataManager.send_table( + output_tables_meta={"table": data_table_meta}, + download_dir=os.path.abspath(parameters.dir_name) + ) + diff --git a/python/fate_flow/manager/container/__init__.py b/python/fate_flow/manager/container/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/manager/container/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/manager/container/docker_manager.py b/python/fate_flow/manager/container/docker_manager.py new file mode 100644 index 000000000..aff4b1bf0 --- /dev/null +++ b/python/fate_flow/manager/container/docker_manager.py @@ -0,0 +1,62 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import docker + +from fate_flow.runtime.component_provider import ComponentProvider + + +class DockerManager: + def __init__(self, provider: ComponentProvider): + self.provider = provider + self.client = docker.DockerClient(base_url=provider.metadata.base_url) + + def start(self, name, command, environment, auto_remove=False, detach=True, network_mode="host", volumes=None): + if not volumes: + volumes = {} + self.client.containers.run( + self.provider.metadata.image, command, + auto_remove=auto_remove, detach=detach, + environment=environment, name=name, + network_mode=network_mode, volumes=volumes + ) + + def stop(self, name): + try: + container = self.client.containers.get(name) + except docker.errors.NotFound: + return + return container.remove(force=True) + + def is_running(self, name): + try: + container = self.client.containers.get(name) + except docker.errors.NotFound: + return False + return container.status == 'running' + + def exit_with_exception(self, name): + try: + container = self.client.containers.get(name) + except docker.errors.NotFound: + return False + return int(container.attrs['State']['ExitCode']) != 0 + + def get_labels(self): + image = self.client.images.get(self.provider.metadata.image) + return image.labels + + def get_env(self): + image = self.client.images.get(self.provider.metadata.image) + return image.attrs.get("Config").get("Env") diff --git a/python/fate_flow/manager/container/k8s_conf_template.yaml b/python/fate_flow/manager/container/k8s_conf_template.yaml new file mode 100644 index 000000000..857b119fe --- /dev/null +++ b/python/fate_flow/manager/container/k8s_conf_template.yaml @@ -0,0 +1,8 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: '' + namespace: fate-9999 +data: + service_conf.yaml: | + \ No newline at end of file diff --git a/python/fate_flow/manager/container/k8s_manager.py b/python/fate_flow/manager/container/k8s_manager.py new file mode 100644 index 000000000..91095a880 --- /dev/null +++ b/python/fate_flow/manager/container/k8s_manager.py @@ -0,0 +1,92 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy +from pathlib import Path + +from kubernetes import client, config +from ruamel import yaml + +from fate_flow.runtime.component_provider import ComponentProvider +from fate_flow.runtime.system_settings import WORKER +from fate_flow.utils.conf_utils import get_base_config +from fate_flow.utils.log import getLogger + +LOGGER = getLogger("k8s-manager") + + +class K8sManager: + image = WORKER.get('k8s', {}).get('image', '') + namespace = WORKER.get('k8s', {}).get('namespace', '') + + def __init__(self, provider: ComponentProvider): + config.load_kube_config() + self.job_template = yaml.safe_load( + (Path(__file__).parent / 'k8s_template.yaml').read_text('utf-8') + ) + self.job_conf_template = yaml.safe_load( + (Path(__file__).parent / 'k8s_conf_template.yaml').read_text('utf-8') + ) + + def populate_yaml_template(self, name, command, environment): + job = copy.deepcopy(self.job_template) + metadata = job['metadata'] + container_spec = job['spec']['template']['spec']['containers'][0] + metadata['name'] = self.convertname(name) + metadata['namespace'] = self.namespace + container_spec['name'] = self.convertname(name) + container_spec['image'] = self.image + container_spec['command'] = ["/data/projects/fate/env/python/venv/bin/python"] + container_spec['args'] = command + container_spec['env'] = [{'name': k, 'value': v} for k, v in environment.items()] + volumes=job['spec']['template']['spec']['volumes'][0] + volumes['configMap']['name']=self.convertname(name + "job-conf") + return job + def populate_conf_yaml_template(self, name, service_conf): + job_conf = copy.deepcopy(self.job_conf_template) + metadata = job_conf['metadata'] + metadata['name'] = self.convertname(name + "job-conf") + metadata['namespace'] = self.namespace + job_conf['data']['service_conf.yaml'] = service_conf + return job_conf + + def start(self, name, command, environment, **kwargs): + # LOGGER.debug(f"command: {type(command)}, {command}") + job = self.populate_yaml_template(self.convertname(name), command, environment) + service_conf=yaml.safe_dump(get_base_config(key=None), default_flow_style=False) + job_conf = self.populate_conf_yaml_template(self.convertname(name), service_conf) + LOGGER.debug(f"job: {job}") + LOGGER.debug(f"job_conf: {job}") + client.CoreV1Api().create_namespaced_config_map(self.namespace, job_conf) + client.BatchV1Api().create_namespaced_job(self.namespace, job) + + def stop(self, name): + LOGGER.debug(f"stop job {name}") + body = client.V1DeleteOptions(propagation_policy='Background') + client.BatchV1Api().delete_namespaced_job(self.convertname(name), self.namespace, body=body, async_req=True) + client.CoreV1Api().delete_namespaced_config_map(self.convertname(name + "job-conf"),self.namespace, body=body, async_req=True) + + def is_running(self, name): + res = client.BatchV1Api().read_namespaced_job_status(self.convertname(name), self.namespace) + # LOGGER.debug(f"res: {res}") + if not res: + return False + return not (res.status.succeeded or res.status.failed) + + def exit_with_exception(self, name): + return False + + # convertname: Ensure that name composes the RFC 1123 specification + def convertname(self, name): + return name.lower().replace('_','-') \ No newline at end of file diff --git a/python/fate_flow/manager/container/k8s_template.yaml b/python/fate_flow/manager/container/k8s_template.yaml new file mode 100644 index 000000000..c6b5509ca --- /dev/null +++ b/python/fate_flow/manager/container/k8s_template.yaml @@ -0,0 +1,33 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: '' + namespace: fate-9999 +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: '' + image: '' + imagePullPolicy: IfNotPresent + command: [] + args: [] + env: [] + volumeMounts: + - mountPath: /data/projects/fate/eggroll/conf/ + name: service-conf + - mountPath: /data/projects/fate/fate_flow/logs/ + name: local-path + subPath: logs + - mountPath: /data/projects/fate/fate_flow/data/ + name: local-path + subPath: data + volumes: + - name: service-conf + configMap: + name: service-conf + - name: local-path + hostPath: + path: /data/projects/fate/fate_flow diff --git a/python/fate_flow/manager/data_manager.py b/python/fate_flow/manager/data_manager.py deleted file mode 100644 index 2c4ebc88b..000000000 --- a/python/fate_flow/manager/data_manager.py +++ /dev/null @@ -1,447 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import copy -import datetime -import json -import operator -import os -import tarfile -import uuid - -from flask import send_file - -from fate_arch import storage -from fate_arch.abc import StorageTableABC -from fate_arch.common.base_utils import fate_uuid -from fate_arch.session import Session -from fate_flow.component_env_utils import feature_utils, env_utils -from fate_flow.settings import stat_logger -from fate_flow.db.db_models import DB, TrackingMetric, DataTableTracking -from fate_flow.utils import data_utils -from fate_flow.utils.base_utils import get_fate_flow_directory -from fate_flow.utils.data_utils import get_header_schema, line_extend_uuid - - -class SchemaMetaParam: - def __init__(self, - delimiter=",", - input_format="dense", - tag_with_value=False, - tag_value_delimiter=":", - with_match_id=False, - id_list=None, - id_range=0, - exclusive_data_type=None, - data_type="float64", - with_label=False, - label_name="y", - label_type="int"): - self.input_format = input_format - self.delimiter = delimiter - self.tag_with_value = tag_with_value - self.tag_value_delimiter = tag_value_delimiter - self.with_match_id = with_match_id - self.id_list = id_list - self.id_range = id_range - self.exclusive_data_type = exclusive_data_type - self.data_type = data_type - self.with_label = with_label - self.label_name = label_name - self.label_type = label_type - self.adapter_param() - - def to_dict(self): - d = {} - for k, v in self.__dict__.items(): - if v is None: - continue - d[k] = v - return d - - def adapter_param(self): - if not self.with_label: - self.label_name = None - self.label_type = None - - -class AnonymousGenerator(object): - @staticmethod - def update_anonymous_header_with_role(schema, role, party_id): - obj = env_utils.get_class_object("anonymous_generator") - return obj.update_anonymous_header_with_role(schema, role, party_id) - - @staticmethod - def generate_anonymous_header(schema): - obj = env_utils.get_class_object("anonymous_generator")() - return obj.generate_anonymous_header(schema) - - @staticmethod - def migrate_schema_anonymous(anonymous_schema, role, party_id, migrate_mapping): - obj = env_utils.get_class_object("anonymous_generator")(role, party_id, migrate_mapping) - return obj.migrate_schema_anonymous(anonymous_schema) - - @staticmethod - def generate_header(computing_table, schema): - obj = env_utils.get_class_object("data_format") - return obj.generate_header(computing_table, schema) - - @staticmethod - def reconstruct_header(schema): - obj = env_utils.get_class_object("data_format") - return obj.reconstruct_header(schema) - - @staticmethod - def recover_schema(schema): - obj = env_utils.get_class_object("data_format") - return obj.recover_schema(schema) - - -class DataTableTracker(object): - @classmethod - @DB.connection_context() - def create_table_tracker(cls, table_name, table_namespace, entity_info): - tracker = DataTableTracking() - tracker.f_table_name = table_name - tracker.f_table_namespace = table_namespace - for k, v in entity_info.items(): - attr_name = 'f_%s' % k - if hasattr(DataTableTracking, attr_name): - setattr(tracker, attr_name, v) - if entity_info.get("have_parent"): - parent_trackers = DataTableTracking.select().where( - DataTableTracking.f_table_name == entity_info.get("parent_table_name"), - DataTableTracking.f_table_namespace == entity_info.get("parent_table_namespace")).order_by(DataTableTracking.f_create_time.desc()) - if not parent_trackers: - tracker.f_source_table_name = entity_info.get("parent_table_name") - tracker.f_source_table_namespace = entity_info.get("parent_table_namespace") - else: - parent_tracker = parent_trackers[0] - if parent_tracker.f_have_parent: - tracker.f_source_table_name = parent_tracker.f_source_table_name - tracker.f_source_table_namespace = parent_tracker.f_source_table_namespace - else: - tracker.f_source_table_name = parent_tracker.f_table_name - tracker.f_source_table_namespace = parent_tracker.f_table_namespace - rows = tracker.save(force_insert=True) - if rows != 1: - raise Exception("Create {} failed".format(tracker)) - return tracker - - @classmethod - @DB.connection_context() - def query_tracker(cls, table_name, table_namespace, is_parent=False): - if not is_parent: - filters = [operator.attrgetter('f_table_name')(DataTableTracking) == table_name, - operator.attrgetter('f_table_namespace')(DataTableTracking) == table_namespace] - else: - filters = [operator.attrgetter('f_parent_table_name')(DataTableTracking) == table_name, - operator.attrgetter('f_parent_table_namespace')(DataTableTracking) == table_namespace] - trackers = DataTableTracking.select().where(*filters) - return [tracker for tracker in trackers] - - - @classmethod - @DB.connection_context() - def get_parent_table(cls, table_name, table_namespace): - trackers = DataTableTracker.query_tracker(table_name, table_namespace) - if not trackers: - raise Exception(f"no found table: table name {table_name}, table namespace {table_namespace}") - else: - parent_table_info = [] - for tracker in trackers: - if not tracker.f_have_parent: - return [] - else: - parent_table_info.append({"parent_table_name": tracker.f_parent_table_name, - "parent_table_namespace": tracker.f_parent_table_namespace, - "source_table_name": tracker.f_source_table_name, - "source_table_namespace": tracker.f_source_table_namespace - }) - return parent_table_info - - @classmethod - @DB.connection_context() - def track_job(cls, table_name, table_namespace, display=False): - trackers = DataTableTracker.query_tracker(table_name, table_namespace, is_parent=True) - job_id_list = [] - for tracker in trackers: - job_id_list.append(tracker.f_job_id) - job_id_list = list(set(job_id_list)) - return {"count": len(job_id_list)} if not display else {"count": len(job_id_list), "job": job_id_list} - - -class TableStorage: - @staticmethod - def collect(src_table, part_of_data): - line_index = 0 - count = 0 - fate_uuid = uuid.uuid1().hex - for k, v in src_table.collect(): - if src_table.meta.get_extend_sid(): - v = src_table.meta.get_id_delimiter().join([k, v]) - k = line_extend_uuid(fate_uuid, line_index) - line_index += 1 - yield k, v - if count <= 100: - part_of_data.append((k, v)) - count += 1 - - @staticmethod - def read(src_table, schema, part_of_data): - line_index = 0 - count = 0 - src_table_meta = src_table.meta - fate_uuid = uuid.uuid1().hex - if src_table_meta.get_have_head(): - get_head = False - else: - get_head = True - if not src_table.meta.get_extend_sid(): - get_line = data_utils.get_data_line - elif not src_table_meta.get_auto_increasing_sid(): - get_line = data_utils.get_sid_data_line - else: - get_line = data_utils.get_auto_increasing_sid_data_line - for line in src_table.read(): - if not get_head: - schema.update(data_utils.get_header_schema( - header_line=line, - id_delimiter=src_table_meta.get_id_delimiter(), - extend_sid=src_table_meta.get_extend_sid(), - )) - get_head = True - continue - values = line.rstrip().split(src_table.meta.get_id_delimiter()) - k, v = get_line( - values=values, - line_index=line_index, - extend_sid=src_table.meta.get_extend_sid(), - auto_increasing_sid=src_table.meta.get_auto_increasing_sid(), - id_delimiter=src_table.meta.get_id_delimiter(), - fate_uuid=fate_uuid, - ) - line_index += 1 - yield k, v - if count <= 100: - part_of_data.append((k, v)) - count += 1 - - @staticmethod - def copy_table(src_table: StorageTableABC, dest_table: StorageTableABC): - part_of_data = [] - src_table_meta = src_table.meta - schema = {} - update_schema = False - if not src_table_meta.get_in_serialized(): - dest_table.put_all(TableStorage.read(src_table, schema, part_of_data)) - else: - source_header = copy.deepcopy(src_table_meta.get_schema().get("header")) - TableStorage.update_full_header(src_table_meta) - dest_table.put_all(TableStorage.collect(src_table, part_of_data)) - schema = src_table.meta.get_schema() - schema["header"] = source_header - if schema.get("extend_tag"): - schema.update({"extend_tag": False}) - _, dest_table.meta = dest_table.meta.update_metas( - schema=schema if not update_schema else None, - part_of_data=part_of_data, - id_delimiter=src_table_meta.get_id_delimiter() - ) - return dest_table.count() - - @staticmethod - def update_full_header(table_meta): - schema = table_meta.get_schema() - if schema.get("anonymous_header"): - header = AnonymousGenerator.reconstruct_header(schema) - schema["header"] = header - table_meta.set_metas(schema=schema) - - @staticmethod - def read_table_data(data_table_meta, limit=100): - if not limit or limit > 100: - limit = 100 - data_table = storage.StorageTableMeta( - name=data_table_meta.get_name(), - namespace=data_table_meta.get_namespace() - ) - if data_table: - table_schema = data_table_meta.get_schema() - out_header = None - data_list = [] - all_extend_header = {} - for k, v in data_table_meta.get_part_of_data(): - data_line, is_str, all_extend_header = feature_utils.get_component_output_data_line( - src_key=k, - src_value=v, - schema=table_schema, - all_extend_header=all_extend_header - ) - data_list.append(data_line) - if len(data_list) == limit: - break - if data_list: - extend_header = feature_utils.generate_header(all_extend_header, schema=table_schema) - out_header = get_component_output_data_schema( - output_table_meta=data_table_meta, - is_str=is_str, - extend_header=extend_header - ) - - return {'header': out_header, 'data': data_list} - - return {'header': [], 'data': []} - - @staticmethod - def send_table(output_tables_meta, tar_file_name="", limit=-1, need_head=True, local_download=False, output_data_file_path=None): - output_data_file_list = [] - output_data_meta_file_list = [] - output_tmp_dir = os.path.join(get_fate_flow_directory(), 'tmp/{}/{}'.format(datetime.datetime.now().strftime("%Y%m%d"), fate_uuid())) - for output_name, output_table_meta in output_tables_meta.items(): - output_data_count = 0 - if not local_download: - output_data_file_path = "{}/{}.csv".format(output_tmp_dir, output_name) - output_data_meta_file_path = "{}/{}.meta".format(output_tmp_dir, output_name) - os.makedirs(os.path.dirname(output_data_file_path), exist_ok=True) - with open(output_data_file_path, 'w') as fw: - with Session() as sess: - output_table = sess.get_table(name=output_table_meta.get_name(), - namespace=output_table_meta.get_namespace()) - all_extend_header = {} - if output_table: - for k, v in output_table.collect(): - data_line, is_str, all_extend_header = feature_utils.get_component_output_data_line( - src_key=k, - src_value=v, - schema=output_table_meta.get_schema(), - all_extend_header=all_extend_header) - # save meta - if output_data_count == 0: - output_data_file_list.append(output_data_file_path) - extend_header = feature_utils.generate_header(all_extend_header, - schema=output_table_meta.get_schema()) - header = get_component_output_data_schema(output_table_meta=output_table_meta, - is_str=is_str, - extend_header=extend_header) - if not local_download: - output_data_meta_file_list.append(output_data_meta_file_path) - with open(output_data_meta_file_path, 'w') as f: - json.dump({'header': header}, f, indent=4) - if need_head and header and output_table_meta.get_have_head() and \ - output_table_meta.get_schema().get("is_display", True): - fw.write('{}\n'.format(','.join(header))) - delimiter = output_table_meta.get_id_delimiter() if output_table_meta.get_id_delimiter() else "," - fw.write('{}\n'.format(delimiter.join(map(lambda x: str(x), data_line)))) - output_data_count += 1 - if output_data_count == limit: - break - if local_download: - return - # tar - output_data_tarfile = "{}/{}".format(output_tmp_dir, tar_file_name) - tar = tarfile.open(output_data_tarfile, mode='w:gz') - for index in range(0, len(output_data_file_list)): - tar.add(output_data_file_list[index], os.path.relpath(output_data_file_list[index], output_tmp_dir)) - tar.add(output_data_meta_file_list[index], - os.path.relpath(output_data_meta_file_list[index], output_tmp_dir)) - tar.close() - for key, path in enumerate(output_data_file_list): - try: - os.remove(path) - os.remove(output_data_meta_file_list[key]) - except Exception as e: - # warning - stat_logger.warning(e) - return send_file(output_data_tarfile, attachment_filename=tar_file_name, as_attachment=True) - - -def delete_tables_by_table_infos(output_data_table_infos): - data = [] - status = False - with Session() as sess: - for output_data_table_info in output_data_table_infos: - table_name = output_data_table_info.f_table_name - namespace = output_data_table_info.f_table_namespace - table_info = {'table_name': table_name, 'namespace': namespace} - if table_name and namespace and table_info not in data: - table = sess.get_table(table_name, namespace) - if table: - try: - table.destroy() - data.append(table_info) - status = True - except Exception as e: - stat_logger.warning(e) - return status, data - - -def delete_metric_data(metric_info): - status = delete_metric_data_from_db(metric_info) - return f"delete status: {status}" - - -@DB.connection_context() -def delete_metric_data_from_db(metric_info): - tracking_metric_model = type(TrackingMetric.model(table_index=metric_info.get("job_id")[:8])) - operate = tracking_metric_model.delete().where(*get_delete_filters(tracking_metric_model, metric_info)) - return operate.execute() > 0 - - -def get_delete_filters(tracking_metric_model, metric_info): - delete_filters = [] - primary_keys = ["job_id"] - for key in primary_keys: - if key in metric_info: - delete_filters.append(operator.attrgetter("f_%s" % key)(tracking_metric_model) == metric_info[key]) - return delete_filters - - -def get_component_output_data_schema(output_table_meta, extend_header, is_str=False) -> list: - # get schema - schema = output_table_meta.get_schema() - if not schema: - return [] - header = [schema.get('sid_name') or schema.get('sid', 'sid')] - if schema.get("extend_tag"): - header = [] - if "label" in extend_header and schema.get("label_name"): - extend_header[extend_header.index("label")] = schema.get("label_name") - header.extend(extend_header) - if is_str or isinstance(schema.get('header'), str): - - if schema.get("original_index_info"): - header = [schema.get('sid_name') or schema.get('sid', 'sid')] - header.extend(AnonymousGenerator.reconstruct_header(schema)) - return header - - if not schema.get('header'): - if schema.get('sid'): - return [schema.get('sid')] - else: - return [] - - if isinstance(schema.get('header'), str): - schema_header = schema.get('header').split(',') - elif isinstance(schema.get('header'), list): - schema_header = schema.get('header') - else: - raise ValueError("header type error") - header.extend([feature for feature in schema_header]) - - else: - header.extend(schema.get('header', [])) - - return header diff --git a/python/fate_flow/manager/dependence_manager.py b/python/fate_flow/manager/dependence_manager.py deleted file mode 100644 index c83a3ca73..000000000 --- a/python/fate_flow/manager/dependence_manager.py +++ /dev/null @@ -1,223 +0,0 @@ -import os -import sys - -from fate_arch import storage -from fate_arch.common import EngineType -from fate_flow.controller.job_controller import JobController -from fate_flow.entity.run_status import JobInheritanceStatus, TaskStatus -from fate_flow.operation.job_saver import JobSaver -from fate_flow.utils.log_utils import schedule_logger -from fate_arch.computing import ComputingEngine -from fate_flow.db.dependence_registry import DependenceRegistry -from fate_flow.entity import ComponentProvider -from fate_flow.entity.types import FateDependenceName, ComponentProviderName, FateDependenceStorageEngine, WorkerName -from fate_flow.manager.provider_manager import ProviderManager -from fate_flow.manager.worker_manager import WorkerManager -from fate_flow.settings import DEPENDENT_DISTRIBUTION, FATE_FLOW_UPDATE_CHECK, ENGINES -from fate_flow.utils import schedule_utils, job_utils, process_utils -from fate_flow.worker.job_inheritor import JobInherit - - -class DependenceManager: - @classmethod - def check_job_dependence(cls, job): - if cls.check_job_inherit_dependence(job) and cls.check_spark_dependence(job): - return True - else: - return False - - @classmethod - def check_job_inherit_dependence(cls, job): - schedule_logger(job.f_job_id).info( - f"check job inherit dependence: {job.f_inheritance_info}, {job.f_inheritance_status}") - if job.f_inheritance_info: - if job.f_inheritance_status == JobInheritanceStatus.WAITING: - cls.start_inheriting_job(job) - return False - elif job.f_inheritance_status == JobInheritanceStatus.RUNNING: - return False - elif job.f_inheritance_status == JobInheritanceStatus.FAILED: - raise Exception("job inheritance failed") - else: - return True - else: - return True - - @classmethod - def component_check(cls, job, check_type="inheritance"): - if check_type == "rerun": - task_list = JobSaver.query_task(job_id=job.f_job_id, party_id=job.f_party_id, role=job.f_role, - status=TaskStatus.SUCCESS, only_latest=True) - tasks = {} - for task in task_list: - tasks[task.f_component_name] = task - else: - tasks = JobController.load_tasks(component_list=job.f_inheritance_info.get("component_list", []), - job_id=job.f_inheritance_info.get("job_id"), - role=job.f_role, - party_id=job.f_party_id) - tracker_dict = JobController.load_task_tracker(tasks) - missing_dependence_component_list = [] - # data dependence - for tracker in tracker_dict.values(): - table_infos = tracker.get_output_data_info() - for table in table_infos: - table_meta = storage.StorageTableMeta(name=table.f_table_name, namespace=table.f_table_namespace) - if not table_meta: - missing_dependence_component_list.append(tracker.component_name) - continue - if check_type == "rerun": - return missing_dependence_component_list - elif check_type == "inheritance": - # reload component list - return list(set(job.f_inheritance_info.get("component_list", [])) - set(missing_dependence_component_list)) - - @classmethod - def start_inheriting_job(cls, job): - JobSaver.update_job(job_info={"job_id": job.f_job_id, "role": job.f_role, "party_id": job.f_party_id, - "inheritance_status": JobInheritanceStatus.RUNNING}) - conf_dir = job_utils.get_job_directory(job_id=job.f_job_id) - os.makedirs(conf_dir, exist_ok=True) - process_cmd = [ - sys.executable or 'python3', - sys.modules[JobInherit.__module__].__file__, - '--job_id', job.f_job_id, - '--role', job.f_role, - '--party_id', job.f_party_id, - ] - log_dir = os.path.join(job_utils.get_job_log_directory(job_id=job.f_job_id), "job_inheritance") - process_utils.run_subprocess(job_id=job.f_job_id, config_dir=conf_dir, process_cmd=process_cmd, - log_dir=log_dir, process_name="job_inheritance") - - @classmethod - def check_spark_dependence(cls, job): - if not DEPENDENT_DISTRIBUTION: - return True - engine_name = ENGINES.get(EngineType.COMPUTING) - schedule_logger(job.f_job_id).info(f"job engine name: {engine_name}") - if engine_name not in [ComputingEngine.SPARK]: - return True - dsl_parser = schedule_utils.get_job_dsl_parser(dsl=job.f_dsl, runtime_conf=job.f_runtime_conf, - train_runtime_conf=job.f_train_runtime_conf) - provider_group = ProviderManager.get_job_provider_group(dsl_parser=dsl_parser, - runtime_conf=job.f_runtime_conf_on_party, - role=job.f_role, - party_id=job.f_party_id) - version_provider_info = {} - fate_flow_version_provider_info = {} - schedule_logger(job.f_job_id).info(f'group_info:{provider_group}') - for group_key, group_info in provider_group.items(): - if group_info["provider"]["name"] == ComponentProviderName.FATE_FLOW.value and \ - group_info["provider"]["version"] not in fate_flow_version_provider_info: - fate_flow_version_provider_info[group_info["provider"]["version"]] = group_info["provider"] - if group_info["provider"]["name"] == ComponentProviderName.FATE.value and \ - group_info["provider"]["version"] not in version_provider_info: - version_provider_info[group_info["provider"]["version"]] = group_info["provider"] - schedule_logger(job.f_job_id).info(f'version_provider_info:{version_provider_info}') - schedule_logger(job.f_job_id).info(f'fate_flow_version_provider_info:{fate_flow_version_provider_info}') - if not version_provider_info: - version_provider_info = fate_flow_version_provider_info - check_tag, upload_tag, upload_details = cls.check_upload(job.f_job_id, version_provider_info, - fate_flow_version_provider_info) - if upload_tag: - cls.upload_spark_dependence(job, upload_details) - return check_tag - - @classmethod - def check_upload(cls, job_id, provider_group, fate_flow_version_provider_info, - storage_engine=FateDependenceStorageEngine.HDFS.value): - schedule_logger(job_id).info("start Check if need to upload dependencies") - schedule_logger(job_id).info(f"{provider_group}") - upload_details = {} - check_tag = True - upload_total = 0 - for version, provider_info in provider_group.items(): - upload_details[version] = {} - provider = ComponentProvider(**provider_info) - for dependence_type in [FateDependenceName.Fate_Source_Code.value, FateDependenceName.Python_Env.value]: - schedule_logger(job_id).info(f"{dependence_type}") - dependencies_storage_info = DependenceRegistry.get_dependencies_storage_meta( - storage_engine=storage_engine, - version=provider.version, - type=dependence_type, - get_or_one=True - ) - need_upload = False - if dependencies_storage_info: - if dependencies_storage_info.f_upload_status: - # version dependence uploading - check_tag = False - continue - elif not dependencies_storage_info.f_storage_path: - need_upload = True - upload_total += 1 - - elif dependence_type == FateDependenceName.Fate_Source_Code.value: - if provider.name == ComponentProviderName.FATE.value: - check_fate_flow_provider_status = False - if fate_flow_version_provider_info.values(): - flow_provider = ComponentProvider(**list(fate_flow_version_provider_info.values())[0]) - check_fate_flow_provider_status = DependenceRegistry.get_modify_time(flow_provider.path) \ - != dependencies_storage_info.f_fate_flow_snapshot_time - if FATE_FLOW_UPDATE_CHECK and check_fate_flow_provider_status: - need_upload = True - upload_total += 1 - elif DependenceRegistry.get_modify_time(provider.path) != \ - dependencies_storage_info.f_snapshot_time: - need_upload = True - upload_total += 1 - elif provider.name == ComponentProviderName.FATE_FLOW.value and FATE_FLOW_UPDATE_CHECK: - if DependenceRegistry.get_modify_time(provider.path) != \ - dependencies_storage_info.f_fate_flow_snapshot_time: - need_upload = True - upload_total += 1 - else: - need_upload = True - upload_total += 1 - if need_upload: - upload_details[version][dependence_type] = provider - if upload_total > 0: - check_tag = False - schedule_logger(job_id).info(f"check dependencies result: {check_tag}, {upload_details}") - return check_tag, upload_total > 0, upload_details - - @classmethod - def upload_spark_dependence(cls, job, upload_details, storage_engine=FateDependenceStorageEngine.HDFS.value): - schedule_logger(job.f_job_id).info(f"start upload dependence: {upload_details}") - for version, type_provider in upload_details.items(): - for dependence_type, provider in type_provider.items(): - storage_meta = { - "f_storage_engine": storage_engine, - "f_type": dependence_type, - "f_version": version, - "f_upload_status": True - } - schedule_logger(job.f_job_id).info(f"update dependence storage meta:{storage_meta}") - DependenceRegistry.save_dependencies_storage_meta(storage_meta, status_check=True) - WorkerManager.start_general_worker(worker_name=WorkerName.DEPENDENCE_UPLOAD, job_id=job.f_job_id, - role=job.f_role, party_id=job.f_party_id, provider=provider, - dependence_type=dependence_type, callback=cls.record_upload_process, - callback_param=["dependence_type", "pid", "provider"]) - - @classmethod - def record_upload_process(cls, provider, dependence_type, pid, - storage_engine=FateDependenceStorageEngine.HDFS.value): - storage_meta = { - "f_storage_engine": storage_engine, - "f_type": dependence_type, - "f_version": provider.version, - "f_pid": pid, - "f_upload_status": True - } - DependenceRegistry.save_dependencies_storage_meta(storage_meta) - - @classmethod - def kill_upload_process(cls, version, storage_engine, dependence_type): - storage_meta = { - "f_storage_engine": storage_engine, - "f_type": dependence_type, - "f_version": version, - "f_upload_status": False, - "f_pid": 0 - } - DependenceRegistry.save_dependencies_storage_meta(storage_meta) diff --git a/python/fate_flow/manager/metric_manager.py b/python/fate_flow/manager/metric_manager.py deleted file mode 100644 index 41c52ffa9..000000000 --- a/python/fate_flow/manager/metric_manager.py +++ /dev/null @@ -1,168 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from fate_arch.common.base_utils import current_timestamp, serialize_b64, deserialize_b64 -from fate_flow.utils.log_utils import schedule_logger -from fate_flow.db import db_utils -from fate_flow.db.db_models import (DB, TrackingMetric) -from fate_flow.entity import Metric -from fate_flow.utils import job_utils - - -class MetricManager: - def __init__(self, job_id: str, role: str, party_id: int, - component_name: str, - task_id: str = None, - task_version: int = None): - self.job_id = job_id - self.role = role - self.party_id = party_id - self.component_name = component_name - self.task_id = task_id - self.task_version = task_version - - @DB.connection_context() - def read_metric_data(self, metric_namespace: str, metric_name: str, job_level=False): - metrics = [] - for k, v in self.read_metrics_from_db(metric_namespace, metric_name, 1, job_level): - metrics.append(Metric(key=k, value=v)) - return metrics - - @DB.connection_context() - def insert_metrics_into_db(self, metric_namespace: str, metric_name: str, data_type: int, kv, job_level=False): - try: - model_class = self.get_model_class() - tracking_metric = model_class() - tracking_metric.f_job_id = self.job_id - tracking_metric.f_component_name = (self.component_name if not job_level - else job_utils.PIPELINE_COMPONENT_NAME) - tracking_metric.f_task_id = self.task_id - tracking_metric.f_task_version = self.task_version - tracking_metric.f_role = self.role - tracking_metric.f_party_id = self.party_id - tracking_metric.f_metric_namespace = metric_namespace - tracking_metric.f_metric_name = metric_name - tracking_metric.f_type = data_type - default_db_source = tracking_metric.to_dict() - tracking_metric_data_source = [] - for k, v in kv: - db_source = default_db_source.copy() - db_source['f_key'] = serialize_b64(k) - db_source['f_value'] = serialize_b64(v) - db_source['f_create_time'] = current_timestamp() - tracking_metric_data_source.append(db_source) - db_utils.bulk_insert_into_db(model_class, tracking_metric_data_source) - except Exception as e: - schedule_logger(self.job_id).exception( - "An exception where inserted metric {} of metric namespace: {} to database:\n{}".format( - metric_name, - metric_namespace, - e - )) - - @DB.connection_context() - def read_metrics_from_db(self, metric_namespace: str, metric_name: str, data_type, job_level=False): - metrics = [] - try: - tracking_metric_model = self.get_model_class() - tracking_metrics = tracking_metric_model.select(tracking_metric_model.f_key, - tracking_metric_model.f_value).where( - tracking_metric_model.f_job_id == self.job_id, - tracking_metric_model.f_component_name == (self.component_name if not job_level - else job_utils.PIPELINE_COMPONENT_NAME), - tracking_metric_model.f_role == self.role, - tracking_metric_model.f_party_id == self.party_id, - tracking_metric_model.f_metric_namespace == metric_namespace, - tracking_metric_model.f_metric_name == metric_name, - tracking_metric_model.f_type == data_type - ) - for tracking_metric in tracking_metrics: - yield deserialize_b64(tracking_metric.f_key), deserialize_b64(tracking_metric.f_value) - except Exception as e: - schedule_logger(self.job_id).exception(e) - raise e - return metrics - - @DB.connection_context() - def clean_metrics(self): - tracking_metric_model = self.get_model_class() - operate = tracking_metric_model.delete().where( - tracking_metric_model.f_task_id == self.task_id, - tracking_metric_model.f_task_version == self.task_version, - tracking_metric_model.f_role == self.role, - tracking_metric_model.f_party_id == self.party_id - ) - return operate.execute() > 0 - - @DB.connection_context() - def get_metric_list(self, job_level: bool = False): - metrics = {} - - tracking_metric_model = self.get_model_class() - if tracking_metric_model.table_exists(): - tracking_metrics = tracking_metric_model.select( - tracking_metric_model.f_metric_namespace, - tracking_metric_model.f_metric_name - ).where( - tracking_metric_model.f_job_id == self.job_id, - tracking_metric_model.f_component_name == (self.component_name if not job_level else 'dag'), - tracking_metric_model.f_role == self.role, - tracking_metric_model.f_party_id == self.party_id - ).distinct() - - for tracking_metric in tracking_metrics: - metrics[tracking_metric.f_metric_namespace] = metrics.get(tracking_metric.f_metric_namespace, []) - metrics[tracking_metric.f_metric_namespace].append(tracking_metric.f_metric_name) - - return metrics - - @DB.connection_context() - def read_component_metrics(self): - try: - tracking_metric_model = self.get_model_class() - tracking_metrics = tracking_metric_model.select().where( - tracking_metric_model.f_job_id == self.job_id, - tracking_metric_model.f_component_name == self.component_name, - tracking_metric_model.f_role == self.role, - tracking_metric_model.f_party_id == self.party_id, - tracking_metric_model.f_task_version == self.task_version - ) - return [tracking_metric for tracking_metric in tracking_metrics] - except Exception as e: - schedule_logger(self.job_id).exception(e) - raise e - - @DB.connection_context() - def reload_metric(self, source_metric_manager): - component_metrics = source_metric_manager.read_component_metrics() - for component_metric in component_metrics: - model_class = self.get_model_class() - tracking_metric = model_class() - tracking_metric.f_job_id = self.job_id - tracking_metric.f_component_name = self.component_name - tracking_metric.f_task_id = self.task_id - tracking_metric.f_task_version = self.task_version - tracking_metric.f_role = self.role - tracking_metric.f_party_id = self.party_id - tracking_metric.f_metric_namespace = component_metric.f_metric_namespace - tracking_metric.f_metric_name = component_metric.f_metric_name - tracking_metric.f_type = component_metric.f_type - tracking_metric.f_key = component_metric.f_key - tracking_metric.f_value = component_metric.f_value - tracking_metric.save() - - def get_model_class(self): - return db_utils.get_dynamic_db_model(TrackingMetric, self.job_id) diff --git a/python/fate_flow/operation/__init__.py b/python/fate_flow/manager/operation/__init__.py similarity index 100% rename from python/fate_flow/operation/__init__.py rename to python/fate_flow/manager/operation/__init__.py diff --git a/python/fate_flow/operation/job_saver.py b/python/fate_flow/manager/operation/base_saver.py similarity index 54% rename from python/fate_flow/operation/job_saver.py rename to python/fate_flow/manager/operation/base_saver.py index 865f98f0b..2dde7e452 100644 --- a/python/fate_flow/operation/job_saver.py +++ b/python/fate_flow/manager/operation/base_saver.py @@ -15,40 +15,56 @@ # import operator -import time -import typing +from functools import reduce +from typing import Type, Union, Dict -from fate_arch.common.base_utils import current_timestamp -from fate_flow.db.db_models import DB, Job, Task, DataBaseModel -from fate_flow.entity.run_status import JobStatus, TaskStatus, EndStatus +from fate_flow.db.base_models import DB, BaseModelOperate, DataBaseModel +from fate_flow.db.db_models import Task, Job +from fate_flow.db.schedule_models import ScheduleTask, ScheduleTaskStatus, ScheduleJob +from fate_flow.entity.types import JobStatus, TaskStatus, EndStatus +from fate_flow.errors.server_error import NoFoundJob, NoFoundTask +from fate_flow.utils.base_utils import current_timestamp from fate_flow.utils.log_utils import schedule_logger, sql_logger -from fate_flow.utils import schedule_utils -import peewee -class JobSaver(object): +class BaseSaver(BaseModelOperate): STATUS_FIELDS = ["status", "party_status"] + OPERATION = { + '==': operator.eq, + '<': operator.lt, + '<=': operator.le, + '>': operator.gt, + '>=': operator.ge, + '!=': operator.ne, + '<<': operator.lshift, + '>>': operator.rshift, + '%': operator.mod, + '**': operator.pow, + '^': operator.xor, + '~': operator.inv, + } @classmethod - def create_job(cls, job_info) -> Job: - return cls.create_job_family_entity(Job, job_info) + def _create_job(cls, job_obj, job_info): + return cls._create_entity(job_obj, job_info) @classmethod - def create_task(cls, task_info) -> Task: - return cls.create_job_family_entity(Task, task_info) + def _create_task(cls, task_obj, task_info): + return cls._create_entity(task_obj, task_info) @classmethod @DB.connection_context() - def delete_job(cls, job_id): - Job.delete().where(Job.f_job_id == job_id) + def _delete_job(cls, job_obj, job_id): + _op = job_obj.delete().where(job_obj.f_job_id == job_id) + return _op.execute() > 0 @classmethod - def update_job_status(cls, job_info): + def _update_job_status(cls, job_obj, job_info): schedule_logger(job_info["job_id"]).info("try to update job status to {}".format(job_info.get("status"))) - update_status = cls.update_status(Job, job_info) + update_status = cls._update_status(job_obj, job_info) if update_status: schedule_logger(job_info["job_id"]).info("update job status successfully") - if EndStatus.contains(job_info.get("status")): + if cls.end_status_contains(job_info.get("status")): new_job_info = {} # only update tag for k in ["job_id", "role", "party_id", "tag"]: @@ -56,19 +72,19 @@ def update_job_status(cls, job_info): new_job_info[k] = job_info[k] if not new_job_info.get("tag"): new_job_info["tag"] = "job_end" - cls.update_entity_table(Job, new_job_info) + cls.update_entity_table(job_obj, new_job_info) else: schedule_logger(job_info["job_id"]).warning("update job status does not take effect") return update_status @classmethod - def update_job(cls, job_info): + def _update_job(cls, job_obj, job_info): schedule_logger(job_info["job_id"]).info("try to update job") if "status" in job_info: # Avoid unintentional usage that updates the status del job_info["status"] schedule_logger(job_info["job_id"]).warning("try to update job, pop job status") - update_status = cls.update_entity_table(Job, job_info) + update_status = cls.update_entity_table(job_obj, job_info) if update_status: schedule_logger(job_info.get("job_id")).info(f"job update successfully: {job_info}") else: @@ -76,9 +92,10 @@ def update_job(cls, job_info): return update_status @classmethod - def update_task_status(cls, task_info): - schedule_logger(task_info["job_id"]).info("try to update task {} {} status".format(task_info["task_id"], task_info["task_version"])) - update_status = cls.update_status(Task, task_info) + def _update_task_status(cls, task_obj, task_info): + schedule_logger(task_info["job_id"]).info("try to update task {} {} status".format(task_info["task_id"], + task_info["task_version"])) + update_status = cls._update_status(task_obj, task_info) if update_status: schedule_logger(task_info["job_id"]).info("update task {} {} status successfully: {}".format(task_info["task_id"], task_info["task_version"], task_info)) else: @@ -86,9 +103,10 @@ def update_task_status(cls, task_info): return update_status @classmethod - def update_task(cls, task_info, report=False): - schedule_logger(task_info["job_id"]).info("try to update task {} {}".format(task_info["task_id"], task_info["task_version"])) - update_status = cls.update_entity_table(Task, task_info) + def _update_task(cls, task_obj, task_info, report=False): + schedule_logger(task_info["job_id"]).info("try to update task {} {}".format(task_info["task_id"], + task_info["task_version"])) + update_status = cls.update_entity_table(task_obj, task_info) if task_info.get("error_report") and report: schedule_logger(task_info["job_id"]).error("role {} party id {} task {} error report: {}".format( task_info["role"], task_info["party_id"], task_info["task_id"], task_info["error_report"])) @@ -98,51 +116,9 @@ def update_task(cls, task_info, report=False): schedule_logger(task_info["job_id"]).warning("task {} {} update does not take effect".format(task_info["task_id"], task_info["task_version"])) return update_status - @classmethod - def reload_task(cls, source_task, target_task): - task_info = {"job_id": target_task.f_job_id, "task_id": target_task.f_task_id, "task_version": target_task.f_task_version, - "role": target_task.f_role, "party_id": target_task.f_party_id} - update_info = {} - update_list = ["cmd", "elapsed", "end_date", "end_time", "engine_conf", "party_status", "run_ip", - "run_pid", "start_date", "start_time", "status", "worker_id"] - for k in update_list: - update_info[k] = getattr(source_task, f"f_{k}") - task_info.update(update_info) - schedule_logger(task_info["job_id"]).info("try to update task {} {}".format(task_info["task_id"], task_info["task_version"])) - schedule_logger(task_info["job_id"]).info("update info: {}".format(update_info)) - update_status = cls.update_entity_table(Task, task_info) - if update_status: - cls.update_task_status(task_info) - schedule_logger(task_info["job_id"]).info("task {} {} update successfully".format(task_info["task_id"], task_info["task_version"])) - else: - schedule_logger(task_info["job_id"]).warning("task {} {} update does not take effect".format(task_info["task_id"], task_info["task_version"])) - return update_status - @classmethod @DB.connection_context() - def create_job_family_entity(cls, entity_model, entity_info): - obj = entity_model() - obj.f_create_time = current_timestamp() - for k, v in entity_info.items(): - attr_name = 'f_%s' % k - if hasattr(entity_model, attr_name): - setattr(obj, attr_name, v) - try: - rows = obj.save(force_insert=True) - if rows != 1: - raise Exception("Create {} failed".format(entity_model)) - return obj - except peewee.IntegrityError as e: - if e.args[0] == 1062 or (isinstance(e.args[0], str) and "UNIQUE constraint failed" in e.args[0]): - sql_logger(job_id=entity_info.get("job_id", "fate_flow")).warning(e) - else: - raise Exception("Create {} failed:\n{}".format(entity_model, e)) - except Exception as e: - raise Exception("Create {} failed:\n{}".format(entity_model, e)) - - @classmethod - @DB.connection_context() - def update_status(cls, entity_model: DataBaseModel, entity_info: dict): + def _update_status(cls, entity_model, entity_info: dict): query_filters = [] primary_keys = entity_model.get_primary_keys_name() for p_k in primary_keys: @@ -159,18 +135,24 @@ def update_status(cls, entity_model: DataBaseModel, entity_info: dict): for status_field in cls.STATUS_FIELDS: if entity_info.get(status_field) and hasattr(entity_model, f"f_{status_field}"): if status_field in ["status", "party_status"]: + # update end time + if hasattr(obj, "f_start_time") and obj.f_start_time: + if cls.end_status_contains(entity_info.get(status_field)): + update_info["end_time"] = current_timestamp() + update_info['elapsed'] = update_info['end_time'] - obj.f_start_time update_info[status_field] = entity_info[status_field] old_status = getattr(obj, f"f_{status_field}") new_status = update_info[status_field] if_pass = False - if isinstance(obj, Task): - if TaskStatus.StateTransitionRule.if_pass(src_status=old_status, dest_status=new_status): + if isinstance(obj, Task) or isinstance(obj, ScheduleTask) or isinstance(obj, ScheduleTaskStatus): + if cls.check_task_status(old_status, new_status): if_pass = True - elif isinstance(obj, Job): - if JobStatus.StateTransitionRule.if_pass(src_status=old_status, dest_status=new_status): + elif isinstance(obj, Job) or isinstance(obj, ScheduleJob): + if cls.check_job_status(old_status, new_status): if_pass = True - if EndStatus.contains(new_status) and new_status not in {JobStatus.SUCCESS, JobStatus.CANCELED}: - update_filters.append(Job.f_rerun_signal == False) + if cls.end_status_contains(new_status) and new_status not in {JobStatus.SUCCESS, JobStatus.CANCELED}: + if isinstance(obj, ScheduleJob): + update_filters.append(ScheduleJob.f_rerun_signal==False) if if_pass: update_filters.append(operator.attrgetter(f"f_{status_field}")(type(obj)) == old_status) else: @@ -179,17 +161,38 @@ def update_status(cls, entity_model: DataBaseModel, entity_info: dict): return cls.execute_update(old_obj=obj, model=entity_model, update_info=update_info, update_filters=update_filters) + @classmethod + def check_task_status(cls, old_status, dest_status): + return TaskStatus.StateTransitionRule.if_pass(src_status=old_status, dest_status=dest_status) + + @classmethod + def check_job_status(cls, old_status, dest_status): + return JobStatus.StateTransitionRule.if_pass(src_status=old_status, dest_status=dest_status) + + @classmethod + def end_status_contains(cls, status): + return EndStatus.contains(status) + @classmethod @DB.connection_context() - def update_entity_table(cls, entity_model, entity_info): + def update_entity_table(cls, entity_model, entity_info, filters: list = None): query_filters = [] primary_keys = entity_model.get_primary_keys_name() - for p_k in primary_keys: - query_filters.append(operator.attrgetter(p_k)(entity_model) == entity_info[p_k.lstrip("f").lstrip("_")]) + if not filters: + for p_k in primary_keys: + query_filters.append(operator.attrgetter(p_k)(entity_model) == entity_info[p_k.lstrip("f").lstrip("_")]) + else: + for _k in filters: + p_k = f"f_{_k}" + query_filters.append(operator.attrgetter(p_k)(entity_model) == entity_info[_k]) objs = entity_model.select().where(*query_filters) if objs: obj = objs[0] else: + if entity_model.__name__ == Job.__name__: + raise NoFoundJob() + if entity_model.__name__ == Job.__name__: + raise NoFoundTask() raise Exception("can not found the {}".format(entity_model.__name__)) update_filters = query_filters[:] update_info = {} @@ -224,46 +227,24 @@ def execute_update(cls, old_obj, model, update_info, update_filters): @classmethod @DB.connection_context() - def query_job(cls, reverse=None, order_by=None, **kwargs): - return Job.query(reverse=reverse, order_by=order_by, **kwargs) + def _query_job(cls, job_obj, reverse=None, order_by=None, **kwargs): + return job_obj.query(reverse=reverse, order_by=order_by, **kwargs) @classmethod @DB.connection_context() - def get_tasks_asc(cls, job_id, role, party_id): - tasks = Task.query(order_by="create_time", reverse=False, job_id=job_id, role=role, party_id=party_id) - tasks_group = cls.get_latest_tasks(tasks=tasks) - return tasks_group - - @classmethod - @DB.connection_context() - def query_task(cls, only_latest=True, reverse=None, order_by=None, **kwargs) -> typing.List[Task]: - tasks = Task.query(reverse=reverse, order_by=order_by, **kwargs) + def _query_task(cls, task_obj, only_latest=True, reverse=None, order_by=None, scheduler_status=False, **kwargs): + tasks = task_obj.query(reverse=reverse, order_by=order_by, **kwargs) if only_latest: - tasks_group = cls.get_latest_tasks(tasks=tasks) + tasks_group = cls.get_latest_tasks(tasks=tasks, scheduler_status=scheduler_status) return list(tasks_group.values()) else: return tasks @classmethod - @DB.connection_context() - def check_task(cls, job_id, role, party_id, components: list): - filters = [ - Task.f_job_id == job_id, - Task.f_role == role, - Task.f_party_id == party_id, - Task.f_component_name << components - ] - tasks = Task.select().where(*filters) - if tasks and len(tasks) == len(components): - return True - else: - return False - - @classmethod - def get_latest_tasks(cls, tasks): + def get_latest_tasks(cls, tasks, scheduler_status=False): tasks_group = {} for task in tasks: - task_key = cls.task_key(task_id=task.f_task_id, role=task.f_role, party_id=task.f_party_id) + task_key = cls.task_key(task_id=task.f_task_id, role=task.f_role, party_id=task.f_party_id) if not scheduler_status else task.f_task_name if task_key not in tasks_group: tasks_group[task_key] = task elif task.f_task_version > tasks_group[task_key].f_task_version: @@ -272,20 +253,58 @@ def get_latest_tasks(cls, tasks): return tasks_group @classmethod - def fill_job_inference_dsl(cls, job_id, role, party_id, dsl_parser, origin_inference_dsl): - # must fill dsl for fate serving - components_parameters = {} - tasks = cls.query_task(job_id=job_id, role=role, party_id=party_id, only_latest=True) + def task_key(cls, task_id, role, party_id): + return f"{task_id}_{role}_{party_id}" + + @classmethod + def get_latest_scheduler_tasks(cls, tasks): + tasks_group = {} for task in tasks: - components_parameters[task.f_component_name] = task.f_component_parameters - return schedule_utils.fill_inference_dsl(dsl_parser, origin_inference_dsl=origin_inference_dsl, components_parameters=components_parameters) + if task.f_task_id not in tasks_group: + tasks_group[task.f_task_id] = task + elif task.f_task_version > tasks_group[task.f_task_id].f_task_version: + tasks_group[task.f_task_id] = task + return tasks_group @classmethod - def task_key(cls, task_id, role, party_id): - return f"{task_id}_{role}_{party_id}" + @DB.connection_context() + def _list(cls, model: Type[DataBaseModel], limit: int = 0, offset: int = 0, + query: dict = None, order_by: Union[str, list, tuple] = None): + data = model.select() + if query: + data = data.where(cls.query_dict2expression(model, query)) + count = data.count() + + if not order_by: + order_by = 'create_time' + if not isinstance(order_by, (list, tuple)): + order_by = (order_by, 'asc') + order_by, order = order_by + order_by = getattr(model, f'f_{order_by}') + order_by = getattr(order_by, order)() + data = data.order_by(order_by) + + if limit > 0: + data = data.limit(limit) + if offset > 0: + data = data.offset(offset) + return list(data), count + + @classmethod + def query_dict2expression(cls, model: Type[DataBaseModel], query: Dict[str, Union[bool, int, str, list, tuple]]): + expression = [] + for field, value in query.items(): + if not isinstance(value, (list, tuple)): + value = ('==', value) + op, *val = value + + field = getattr(model, f'f_{field}') + value = cls.OPERATION[op](field, val[0]) if op in cls.OPERATION else getattr(field, op)(*val) + + expression.append(value) + return reduce(operator.iand, expression) -def str_to_time_stamp(time_str): - time_array = time.strptime(time_str, "%Y-%m-%d %H:%M:%S") - time_stamp = int(time.mktime(time_array) * 1000) - return time_stamp + @property + def supported_operators(self): + return \ No newline at end of file diff --git a/python/fate_flow/manager/operation/job_saver.py b/python/fate_flow/manager/operation/job_saver.py new file mode 100644 index 000000000..53f32b654 --- /dev/null +++ b/python/fate_flow/manager/operation/job_saver.py @@ -0,0 +1,170 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from fate_flow.db.base_models import DB +from fate_flow.db.db_models import Job, Task +from fate_flow.entity.types import PROTOCOL +from fate_flow.errors.server_error import NoFoundTask +from fate_flow.manager.operation.base_saver import BaseSaver +from fate_flow.db.schedule_models import ScheduleJob, ScheduleTask, ScheduleTaskStatus + + +class JobSaver(BaseSaver): + @classmethod + def create_job(cls, job_info) -> Job: + return cls._create_job(Job, job_info) + + @classmethod + def create_task(cls, task_info) -> Task: + return cls._create_task(Task, task_info) + + @classmethod + def delete_job(cls, job_id): + return cls._delete_job(Job, job_id) + + @classmethod + def delete_task(cls, job_id): + return cls._delete_job(Task, job_id) + + @classmethod + def update_job_status(cls, job_info): + return cls._update_job_status(Job, job_info) + + @classmethod + def query_job(cls, reverse=None, order_by=None, **kwargs): + return cls._query_job(Job, reverse, order_by, **kwargs) + + @classmethod + def update_job(cls, job_info): + return cls._update_job(Job, job_info) + + @classmethod + def update_job_user(cls, job_id, user_name): + return cls.update_entity_table(Job, { + "job_id": job_id, + "user_name": user_name + }, filters=["job_id"]) + + @classmethod + def list_job(cls, limit, offset, query, order_by): + return cls._list(Job, limit, offset, query, order_by) + + @classmethod + def list_task(cls, limit, offset, query, order_by): + return cls._list(Task, limit, offset, query, order_by) + + @classmethod + def query_task( + cls, only_latest=True, reverse=None, order_by=None, ignore_protocol=False, protocol=PROTOCOL.FATE_FLOW, + **kwargs + ): + if not ignore_protocol: + kwargs["protocol"] = protocol + return cls._query_task( + Task, only_latest=only_latest, reverse=reverse, order_by=order_by, **kwargs + ) + + @classmethod + def query_task_by_execution_id(cls, execution_id): + tasks = cls.query_task(execution_id=execution_id) + if not tasks: + raise NoFoundTask(execution_id=execution_id) + return tasks[0] + + @classmethod + def update_task_status(cls, task_info): + return cls._update_task_status(Task, task_info) + + @classmethod + def update_task(cls, task_info, report=False): + return cls._update_task(Task, task_info, report) + + @classmethod + def task_key(cls, task_id, role, party_id): + return f"{task_id}_{role}_{party_id}" + + +class ScheduleJobSaver(BaseSaver): + @classmethod + def create_job(cls, job_info) -> ScheduleJob: + return cls._create_job(ScheduleJob, job_info) + + @classmethod + def create_task(cls, task_info) -> ScheduleTask: + return cls._create_task(ScheduleTask, task_info) + + @classmethod + def delete_job(cls, job_id): + return cls._delete_job(ScheduleJob, job_id) + + @classmethod + def update_job_status(cls, job_info): + return cls._update_job_status(ScheduleJob, job_info) + + @classmethod + def update_job(cls, job_info): + return cls._update_job(ScheduleJob, job_info) + + @classmethod + def query_job(cls, reverse=None, order_by=None, protocol=PROTOCOL.FATE_FLOW, **kwargs): + return cls._query_job(ScheduleJob, reverse, order_by, protocol=protocol, **kwargs) + + @classmethod + def query_task(cls, only_latest=True, reverse=None, order_by=None, scheduler_status=False, **kwargs): + if not scheduler_status: + obj = ScheduleTask + else: + obj = ScheduleTaskStatus + return cls._query_task(obj, only_latest=only_latest, reverse=reverse, order_by=order_by, + scheduler_status=scheduler_status, **kwargs) + + @classmethod + def update_task_status(cls, task_info, scheduler_status=False): + if "party_status" in task_info: + task_info["status"] = task_info["party_status"] + task_obj = ScheduleTask + if scheduler_status: + task_obj = ScheduleTaskStatus + return cls._update_task_status(task_obj, task_info) + + @classmethod + def update_task(cls, task_info, report=False): + cls._update_task(ScheduleTaskStatus, task_info, report) + + @classmethod + @DB.connection_context() + def get_status_tasks_asc(cls, job_id): + tasks = ScheduleTaskStatus.query(order_by="create_time", reverse=False, job_id=job_id) + tasks_group = cls.get_latest_tasks(tasks=tasks, scheduler_status=True) + return tasks_group + + @classmethod + def task_key(cls, task_id, role, party_id): + return f"{task_id}_{role}_{party_id}" + + @classmethod + def create_task_scheduler_status(cls, task_info): + cls._create_entity(ScheduleTaskStatus, task_info) + + @classmethod + def get_latest_scheduler_tasks(cls, tasks): + tasks_group = {} + for task in tasks: + if task.f_task_id not in tasks_group: + tasks_group[task.f_task_id] = task + elif task.f_task_version > tasks_group[task.f_task_id].f_task_version: + tasks_group[task.f_task_id] = task + return tasks_group diff --git a/python/fate_flow/manager/outputs/__init__.py b/python/fate_flow/manager/outputs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/manager/outputs/data.py b/python/fate_flow/manager/outputs/data.py new file mode 100644 index 000000000..48832ca42 --- /dev/null +++ b/python/fate_flow/manager/outputs/data.py @@ -0,0 +1,317 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import os +import tarfile +import uuid +from tempfile import TemporaryDirectory +from typing import List + +from flask import send_file + +from fate_flow.db import TrackingOutputInfo +from fate_flow.db.base_models import BaseModelOperate +from fate_flow.engine import storage +from fate_flow.engine.storage import Session, StorageEngine, DataType +from fate_flow.entity.types import EggRollAddress, StandaloneAddress, HDFSAddress, PathAddress, ApiAddress +from fate_flow.errors.server_error import NoFoundTable +from fate_flow.runtime.system_settings import LOCALFS_DATA_HOME, STANDALONE_DATA_HOME, STORAGE +from fate_flow.utils import job_utils +from fate_flow.utils.io_utils import URI +from fate_flow.utils.wraps_utils import filter_parameters + + +class OutputDataTracking(BaseModelOperate): + @classmethod + def create(cls, entity_info): + cls._create_entity(TrackingOutputInfo, entity_info) + + @classmethod + @filter_parameters() + def query(cls, reverse=False, **kwargs) -> List[TrackingOutputInfo]: + return cls._query(TrackingOutputInfo, reverse=reverse, order_by="index", **kwargs) + + +class DataManager: + @classmethod + def send_table( + cls, + output_tables_meta, + tar_file_name="", + need_head=True, + download_dir="", + + ): + if not need_head: + need_head = True + output_data_file_list = [] + output_data_meta_file_list = [] + with TemporaryDirectory() as output_tmp_dir: + for output_name, output_table_metas in output_tables_meta.items(): + if not isinstance(output_table_metas, list): + output_table_metas = [output_table_metas] + for index, output_table_meta in enumerate(output_table_metas): + if not download_dir: + output_data_file_path = "{}/{}/{}.csv".format(output_tmp_dir, output_name, index) + output_data_meta_file_path = "{}/{}/{}.meta".format(output_tmp_dir, output_name, index) + else: + output_data_file_path = "{}/{}/{}.csv".format(download_dir, output_name, index) + output_data_meta_file_path = "{}/{}/{}.meta".format(download_dir, output_name, index) + output_data_file_list.append(output_data_file_path) + output_data_meta_file_list.append(output_data_meta_file_path) + os.makedirs(os.path.dirname(output_data_file_path), exist_ok=True) + with Session() as sess: + if not output_table_meta: + raise NoFoundTable() + table = sess.get_table( + name=output_table_meta.get_name(), + namespace=output_table_meta.get_namespace()) + cls.write_data_to_file(output_data_file_path, output_data_meta_file_path, table, need_head) + if download_dir: + return + # tar + output_data_tarfile = "{}/{}".format(output_tmp_dir, tar_file_name) + tar = tarfile.open(output_data_tarfile, mode='w:gz') + for index in range(0, len(output_data_file_list)): + tar.add(output_data_file_list[index], os.path.relpath(output_data_file_list[index], output_tmp_dir)) + tar.add(output_data_meta_file_list[index], + os.path.relpath(output_data_meta_file_list[index], output_tmp_dir)) + tar.close() + return send_file(output_data_tarfile, download_name=tar_file_name, as_attachment=True, mimetype='application/gzip') + + @classmethod + def write_data_to_file(cls, output_data_file_path, output_data_meta_file_path, table, need_head): + with open(output_data_file_path, 'w') as fw: + data_meta = table.meta.get_data_meta() + header = cls.get_data_header(table.meta.get_id_delimiter(), data_meta) + with open(output_data_meta_file_path, 'w') as f: + json.dump({'header': header}, f, indent=4) + if table: + write_header = False + for v in cls.collect_data(table=table): + # save meta + if not write_header and need_head and header and table.meta.get_have_head(): + if isinstance(header, list): + header = table.meta.get_id_delimiter().join(header) + fw.write(f'{header}\n') + write_header = True + delimiter = table.meta.get_id_delimiter() + if isinstance(v, str): + fw.write('{}\n'.format(v)) + elif isinstance(v, list): + fw.write('{}\n'.format(delimiter.join([str(_v) for _v in v]))) + else: + raise ValueError(f"type={type(v)}, v={v}") + + @staticmethod + def collect_data(table): + if table.data_type == DataType.DATAFRAME: + for _, data in table.collect(): + for v in data: + yield v + elif table.data_type == DataType.TABLE: + for _k, _v in table.collect(): + yield table.meta.get_id_delimiter().join([_k, _v]) + else: + return [] + + @staticmethod + def display_data(table_metas): + datas = {} + for key, metas in table_metas.items(): + datas[key] = [] + for meta in metas: + if meta.data_type in [DataType.DATAFRAME, DataType.TABLE]: + datas[key].append({"data": meta.get_part_of_data(), "metadata": meta.get_data_meta()}) + else: + continue + return datas + + @classmethod + def query_output_data_table(cls, **kwargs): + data_list = OutputDataTracking.query(**kwargs) + outputs = {} + for data in data_list: + if data.f_output_key not in outputs: + outputs[data.f_output_key] = [] + outputs[data.f_output_key].append({"namespace": data.f_namespace, "name": data.f_name}) + return outputs + + @classmethod + def download_output_data(cls, tar_file_name, **kwargs): + outputs = {} + for key, tables in cls.query_output_data_table(**kwargs).items(): + if key not in outputs: + outputs[key] = [] + for table in tables: + outputs[key].append(storage.StorageTableMeta( + name=table.get("name"), + namespace=table.get("namespace") + )) + + if not outputs: + raise NoFoundTable() + + return cls.send_table(outputs, tar_file_name=tar_file_name) + + @classmethod + def display_output_data(cls, **kwargs): + outputs = {} + for key, tables in cls.query_output_data_table(**kwargs).items(): + if key not in outputs: + outputs[key] = [] + for table in tables: + outputs[key].append(storage.StorageTableMeta( + name=table.get("name"), + namespace=table.get("namespace") + )) + return cls.display_data(outputs) + + @staticmethod + def delete_data(namespace, name): + with Session() as sess: + table = sess.get_table(name=name, namespace=namespace) + if table: + table.destroy() + return True + return False + + @staticmethod + def create_data_table( + namespace, name, uri, partitions, data_meta, data_type, part_of_data=None, count=None, source=None + ): + engine, address = DataManager.uri_to_address(uri) + storage_meta = storage.StorageTableBase( + namespace=namespace, name=name, address=address, + partitions=partitions, engine=engine, + options=None, + key_serdes_type=0, + value_serdes_type=0, + partitioner_type=0, + ) + storage_meta.create_meta( + data_meta=data_meta, part_of_data=part_of_data, count=count, source=source, data_type=data_type + ) + + @staticmethod + def uri_to_address(uri): + uri_schema = URI.from_string(uri).to_schema() + engine = uri_schema.schema() + if engine == StorageEngine.EGGROLL: + address = EggRollAddress(namespace=uri_schema.namespace, name=uri_schema.name) + elif uri_schema.schema() == StorageEngine.STANDALONE: + address = StandaloneAddress(namespace=uri_schema.namespace, name=uri_schema.name) + elif uri_schema.schema() == StorageEngine.HDFS: + address = HDFSAddress(path=uri_schema.path, name_node=uri_schema.authority) + elif uri_schema.schema() in [StorageEngine.PATH, StorageEngine.FILE]: + address = PathAddress(path=uri_schema.path) + elif uri_schema.schema() in [StorageEngine.HTTP]: + address = ApiAddress(url=uri_schema.path) + else: + raise ValueError(f"uri {uri} engine could not be converted to an address") + return engine, address + + @staticmethod + def get_data_info(namespace, name): + data_table_meta = storage.StorageTableMeta(name=name, namespace=namespace) + if data_table_meta: + data = { + "namespace": namespace, + "name": name, + "count": data_table_meta.count, + "meta": data_table_meta.get_data_meta(), + "engine": data_table_meta.engine, + "path": data_table_meta.address.engine_path, + "source": data_table_meta.source, + "data_type": data_table_meta.data_type + } + display_data = data_table_meta.part_of_data + return data, display_data + raise NoFoundTable(name=name, namespace=namespace) + + @staticmethod + def get_data_header(delimiter, data_meta): + header = [] + if data_meta.get("header"): + header = data_meta.get("header") + if isinstance(header, str): + header = header.split(delimiter) + else: + for field in data_meta.get("schema_meta", {}).get("fields", []): + header.append(field.get("name")) + return header + + +class DatasetManager: + @staticmethod + def task_output_name(task_id, task_version): + return f"output_data_{task_id}_{task_version}", uuid.uuid1().hex + + @staticmethod + def get_output_name(uri): + namespace, name = uri.split("/")[-2], uri.split("/")[-1] + return namespace, name + + @classmethod + def upload_data_path(cls, name, namespace, prefix=None, storage_engine=StorageEngine.HDFS): + if storage_engine == StorageEngine.HDFS: + return cls.default_hdfs_path(data_type="input", name=name, namespace=namespace, prefix=prefix) + elif storage_engine == StorageEngine.FILE: + return cls.default_localfs_path(data_type="input", name=name, namespace=namespace) + + @classmethod + def output_data_uri(cls, storage_engine, task_id, is_multi=False): + if storage_engine == StorageEngine.STANDALONE: + uri = f"{storage_engine}://{STANDALONE_DATA_HOME}/{task_id}/{uuid.uuid1().hex}" + elif storage_engine == StorageEngine.HDFS: + uri = cls.default_output_fs_path(uuid.uuid1().hex, task_id, storage_engine=storage_engine) + elif storage_engine == StorageEngine.FILE: + uri = f"file://{cls.default_output_fs_path(uuid.uuid1().hex, task_id, storage_engine=storage_engine)}" + else: + # egg: eggroll + uri = f"{storage_engine}:///{task_id}/{uuid.uuid1().hex}" + + if is_multi: + # replace "{index}" + uri += "_{index}" + return uri + + @classmethod + def output_local_uri(cls, name, type_name, task_info, is_multi=False, base_dir=""): + path = job_utils.get_task_directory(**task_info, output=True, base_dir=base_dir) + uri = os.path.join(f"file://{path}", name, type_name) + if is_multi: + # replace "{index}" + uri += "_{index}" + return uri + + @classmethod + def default_output_fs_path(cls, name, namespace, prefix=None, storage_engine=StorageEngine.HDFS): + if storage_engine == StorageEngine.HDFS: + return f'{STORAGE.get(storage_engine).get("name_node")}' \ + f'{cls.default_hdfs_path(data_type="output", name=name, namespace=namespace, prefix=prefix)}' + elif storage_engine == StorageEngine.FILE: + return cls.default_localfs_path(data_type="output", name=name, namespace=namespace) + + @staticmethod + def default_localfs_path(name, namespace, data_type): + return os.path.join(LOCALFS_DATA_HOME, namespace, name) + + @staticmethod + def default_hdfs_path(data_type, name, namespace, prefix=None): + p = f"/fate/{data_type}/{namespace}/{name}" + if prefix: + p = f"{prefix}/{p}" + return p diff --git a/python/fate_flow/manager/outputs/log.py b/python/fate_flow/manager/outputs/log.py new file mode 100644 index 000000000..211039d6a --- /dev/null +++ b/python/fate_flow/manager/outputs/log.py @@ -0,0 +1,90 @@ +import os +import subprocess + +from fate_flow.runtime.system_settings import LOG_DIR +from fate_flow.utils.log_utils import replace_ip + +JOB = ["schedule_info", "schedule_error"] +TASK = ["task_error", "task_info", "task_warning", "task_debug"] + + +def parameters_check(log_type, job_id, role, party_id): + if log_type in JOB: + if not job_id: + return False + if log_type in TASK: + if not job_id or not role or not party_id: + return False + return True + + +class LogManager: + def __init__(self, log_type, job_id, party_id="", role="", task_name="", **kwargs): + self.log_type = log_type + self.job_id = job_id + self.party_id = party_id + self.role = role + self.task_name = task_name + + @property + def task_base_path(self): + if self.role and self.party_id: + path = os.path.join(self.job_id, self.role, self.party_id) + if self.task_name: + path = os.path.join(path, self.task_name, 'root') + return path + return "" + + @property + def file_path(self): + status = parameters_check(self.log_type, self.job_id, self.role, self.party_id) + if not status: + raise Exception(f"job type {self.log_type} Missing parameters") + type_dict = { + "schedule_info": os.path.join(self.job_id, "fate_flow_schedule.log"), + "schedule_error": os.path.join(self.job_id, "fate_flow_schedule_error.log"), + "task_error": os.path.join(self.task_base_path, "ERROR"), + "task_warning": os.path.join(self.task_base_path, "WARNING"), + "task_info": os.path.join(self.task_base_path, "INFO"), + "task_debug": os.path.join(self.task_base_path, "DEBUG") + } + if self.log_type not in type_dict.keys(): + raise Exception(f"no found log type {self.log_type}") + return os.path.join(LOG_DIR, type_dict[self.log_type]) + + def cat_log(self, begin, end): + line_list = [] + log_path = self.file_path + if begin and end: + cmd = f"cat {log_path} | tail -n +{begin}| head -n {end-begin+1}" + elif begin: + cmd = f"cat {log_path} | tail -n +{begin}" + elif end: + cmd = f"cat {log_path} | head -n {end}" + else: + cmd = f"cat {log_path}" + lines = self.execute(cmd) + if lines: + line_list = [] + line_num = begin if begin else 1 + for line in lines.split("\n"): + line = replace_ip(line) + line_list.append({"line_num": line_num, "content": line}) + line_num += 1 + return line_list + + def count(self): + try: + if os.path.exists(self.file_path): + return int(self.execute(f"cat {self.file_path} | wc -l").strip()) + return 0 + except: + return 0 + + @staticmethod + def execute(cmd): + res = subprocess.run( + cmd, shell=True, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + ) + return res.stdout diff --git a/python/fate_flow/manager/outputs/metric.py b/python/fate_flow/manager/outputs/metric.py new file mode 100644 index 000000000..b2bc61d03 --- /dev/null +++ b/python/fate_flow/manager/outputs/metric.py @@ -0,0 +1,135 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import operator +from typing import List + +from fate_flow.db.base_models import DB +from fate_flow.db.db_models import Metric +from fate_flow.entity.spec.dag import MetricData +from fate_flow.utils import db_utils +from fate_flow.utils.log_utils import schedule_logger + + +class OutputMetric: + def __init__(self, job_id: str, role: str, party_id: str, task_name: str, task_id: str = None, + task_version: int = None): + self.job_id = job_id + self.role = role + self.party_id = party_id + self.task_name = task_name + self.task_id = task_id + self.task_version = task_version + + def save_output_metrics(self, data): + if not data or not isinstance(data, list): + raise RuntimeError(f"Save metric data failed, data is {data}") + return self._insert_metrics_into_db( + self.job_id, self.role, self.party_id, self.task_id, self.task_version, self.task_name, data + ) + + def save_as(self, job_id, role, party_id, task_name, task_id, task_version): + data_list = self.read_metrics() + self._insert_metrics_into_db( + job_id, role, party_id, task_id, task_version, task_name, data_list + ) + + @DB.connection_context() + def _insert_metrics_into_db(self, job_id, role, party_id, task_id, task_version, task_name, data_list): + model_class = self.get_model_class(job_id) + if not model_class.table_exists(): + model_class.create_table() + metric_list = [{ + "f_job_id": job_id, + "f_task_id": task_id, + "f_task_version": task_version, + "f_role": role, + "f_party_id": party_id, + "f_task_name": task_name, + "f_name": data.get("name"), + "f_type": data.get("type"), + "f_groups": data.get("groups"), + "f_step_axis": data.get("step_axis"), + "f_data": data.get("data") + + } for data in data_list] + + with DB.atomic(): + for i in range(0, len(metric_list), 100): + model_class.insert_many(metric_list[i: i+100]).execute() + + @DB.connection_context() + def read_metrics(self, filters_args: dict = None): + try: + if not filters_args: + filters_args = {} + tracking_metric_model = self.get_model_class(self.job_id) + key_list = ["name", "type", "groups", "step_axis"] + filters = [ + tracking_metric_model.f_job_id == self.job_id, + tracking_metric_model.f_role == self.role, + tracking_metric_model.f_party_id == self.party_id, + tracking_metric_model.f_task_id == self.task_id, + tracking_metric_model.f_task_version == self.task_version + ] + for k, v in filters_args.items(): + if k in key_list: + if v is not None: + filters.append(operator.attrgetter(f"f_{k}")(tracking_metric_model) == v) + metrics = tracking_metric_model.select( + tracking_metric_model.f_name, + tracking_metric_model.f_type, + tracking_metric_model.f_groups, + tracking_metric_model.f_step_axis, + tracking_metric_model.f_data + ).where(*filters) + return [metric.to_human_model_dict() for metric in metrics] + except Exception as e: + schedule_logger(self.job_id).exception(e) + raise e + + @DB.connection_context() + def query_metric_keys(self): + try: + tracking_metric_model = self.get_model_class(self.job_id) + metrics = tracking_metric_model.select( + tracking_metric_model.f_name, + tracking_metric_model.f_type, + tracking_metric_model.f_groups, + tracking_metric_model.f_step_axis + ).where( + tracking_metric_model.f_job_id == self.job_id, + tracking_metric_model.f_role == self.role, + tracking_metric_model.f_party_id == self.party_id, + tracking_metric_model.f_task_id == self.task_id, + tracking_metric_model.f_task_version == self.task_version + ).distinct() + return [metric.to_human_model_dict() for metric in metrics] + except Exception as e: + schedule_logger(self.job_id).exception(e) + raise e + + @DB.connection_context() + def delete_metrics(self): + tracking_metric_model = self.get_model_class(self.job_id) + operate = tracking_metric_model.delete().where( + tracking_metric_model.f_task_id == self.task_id, + tracking_metric_model.f_role == self.role, + tracking_metric_model.f_party_id == self.party_id + ) + return operate.execute() > 0 + + @staticmethod + def get_model_class(job_id): + return db_utils.get_dynamic_db_model(Metric, job_id) diff --git a/python/fate_flow/manager/outputs/model/__init__.py b/python/fate_flow/manager/outputs/model/__init__.py new file mode 100644 index 000000000..5425dc19b --- /dev/null +++ b/python/fate_flow/manager/outputs/model/__init__.py @@ -0,0 +1,18 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_flow.manager.outputs.model.model_manager import PipelinedModel +from fate_flow.manager.outputs.model.model_meta import ModelMeta + +__all__ = ["PipelinedModel", "ModelMeta"] diff --git a/python/fate_flow/manager/outputs/model/engine/__init__.py b/python/fate_flow/manager/outputs/model/engine/__init__.py new file mode 100644 index 000000000..a2b1d0bb9 --- /dev/null +++ b/python/fate_flow/manager/outputs/model/engine/__init__.py @@ -0,0 +1,18 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_flow.manager.outputs.model.engine._mysql import MysqlModelStorage +from fate_flow.manager.outputs.model.engine._tencent_cos import TencentCosStorage + +__all__ = ["MysqlModelStorage", "TencentCosStorage"] diff --git a/python/fate_flow/manager/outputs/model/engine/_mysql.py b/python/fate_flow/manager/outputs/model/engine/_mysql.py new file mode 100644 index 000000000..a7539c2fe --- /dev/null +++ b/python/fate_flow/manager/outputs/model/engine/_mysql.py @@ -0,0 +1,141 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import base64 +import io + +from copy import deepcopy + +from peewee import PeeweeException, CharField, IntegerField, CompositeKey +from playhouse.pool import PooledMySQLDatabase + +from fate_flow.db.base_models import LOGGER, BaseModel, LongTextField +from fate_flow.utils.password_utils import decrypt_database_config + +DB = PooledMySQLDatabase(None) +SLICE_MAX_SIZE = 1024 * 1024 * 8 + + +class MachineLearningModel(BaseModel): + f_storage_key = CharField(max_length=100) + f_content = LongTextField(default='') + f_slice_index = IntegerField(default=0) + + class Meta: + database = DB + db_table = 't_machine_learning_model' + primary_key = CompositeKey('f_storage_key', 'f_slice_index') + + +class MysqlModelStorage(object): + def __init__(self, storage_address, decrypt_key=None): + self.init_db(storage_address, decrypt_key) + + def exists(self, storage_key: str): + try: + with DB.connection_context(): + counts = MachineLearningModel.select().where( + MachineLearningModel.f_storage_key == storage_key + ).count() + return counts > 0 + except PeeweeException as e: + # Table doesn't exist + if e.args and e.args[0] == 1146: + return False + + raise e + finally: + self.close_connection() + + def delete(self, storage_key): + if not self.exists(storage_key): + raise FileNotFoundError(f'The model {storage_key} not found in the database.') + return MachineLearningModel.delete().where( + MachineLearningModel.f_storage_key == storage_key + ).execute() + + def store(self, memory_io, storage_key, force_update=True): + memory_io.seek(0) + if not force_update and self.exists(storage_key): + raise FileExistsError(f'The model {storage_key} already exists in the database.') + + try: + DB.create_tables([MachineLearningModel]) + with DB.connection_context(): + MachineLearningModel.delete().where( + MachineLearningModel.f_storage_key == storage_key + ).execute() + + LOGGER.info(f'Starting store model {storage_key}.') + + slice_index = 0 + while True: + content = memory_io.read(SLICE_MAX_SIZE) + if not content: + break + model_in_table = MachineLearningModel() + model_in_table.f_storage_key = storage_key + model_in_table.f_content = base64.b64encode(content) + model_in_table.f_slice_index = slice_index + + rows = model_in_table.save(force_insert=True) + if not rows: + raise IndexError(f'Save slice index {slice_index} failed') + + LOGGER.info(f'Saved slice index {slice_index} of model {storage_key}.') + slice_index += 1 + except Exception as e: + LOGGER.exception(e) + raise Exception(f'Store model {storage_key} to mysql failed.') + else: + LOGGER.info(f'Store model {storage_key} to mysql successfully.') + finally: + self.close_connection() + + def read(self, storage_key): + _io = io.BytesIO() + if not self.exists(storage_key): + raise Exception(f'model {storage_key} not exist in the database.') + try: + with DB.connection_context(): + models_in_tables = MachineLearningModel.select().where( + MachineLearningModel.f_storage_key == storage_key + ).order_by(MachineLearningModel.f_slice_index) + + for model in models_in_tables: + _io.write(base64.b64decode(model.f_content)) + + except Exception as e: + LOGGER.exception(e) + raise Exception(f'read model {storage_key} from mysql failed.') + else: + LOGGER.debug(f'read model from mysql successfully.') + finally: + self.close_connection() + return _io + + @staticmethod + def init_db(storage_address, decrypt_key): + _storage_address = deepcopy(storage_address) + database = _storage_address.pop('name') + decrypt_database_config(_storage_address, decrypt_key=decrypt_key) + DB.init(database, **_storage_address) + + @staticmethod + def close_connection(): + if DB: + try: + DB.close() + except Exception as e: + LOGGER.exception(e) diff --git a/python/fate_flow/manager/outputs/model/engine/_tencent_cos.py b/python/fate_flow/manager/outputs/model/engine/_tencent_cos.py new file mode 100644 index 000000000..e0a2bb0ab --- /dev/null +++ b/python/fate_flow/manager/outputs/model/engine/_tencent_cos.py @@ -0,0 +1,88 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io + +from copy import deepcopy + +from fate_flow.db.base_models import LOGGER + + +class TencentCosStorage(object): + def __init__(self, storage_address, decrypt_key=None): + self.Bucket = storage_address.get("Bucket") + self.client = self.init_client(storage_address) + + def exists(self, storage_key: str): + from qcloud_cos import CosServiceError + + try: + self.client.head_object( + Bucket=self.Bucket, + Key=storage_key, + ) + except CosServiceError as e: + if e.get_error_code() != 'NoSuchResource': + raise e + return False + else: + return True + + def store(self, memory_io, storage_key, force_update=True): + memory_io.seek(0) + if not force_update and self.exists(storage_key): + raise FileExistsError(f'The model {storage_key} already exists in the Cos.') + + try: + rt = self.client.put_object(Bucket=self.Bucket, Key=storage_key, Body=memory_io) + except Exception as e: + raise Exception(f"Store model {storage_key} to Tencent COS failed: {e}") + else: + LOGGER.info(f"Store model {storage_key} to Tencent COS successfully. " + f"ETag: {rt['ETag']}") + + def read(self, storage_key): + _io = io.BytesIO() + try: + rt = self.client.get_object( + Bucket=self.Bucket, + Key=storage_key + ) + _io.write(rt.get("Body").get_raw_stream().read()) + except Exception as e: + LOGGER.exception(e) + raise Exception(f"Read model {storage_key} from Tencent COS failed: {e}") + else: + LOGGER.info(f"Read model {storage_key} from Tencent COS successfully") + return _io + + def delete(self, storage_key): + if not self.exists(storage_key): + raise FileExistsError(f'The model {storage_key} not exist in the Cos.') + try: + rt = self.client.delete_bucket( + Bucket=self.Bucket, + Key=storage_key + ) + except Exception as e: + LOGGER.exception(e) + raise Exception(f"Delete model {storage_key} from Tencent COS failed: {e}") + + @staticmethod + def init_client(storage_address): + from qcloud_cos import CosS3Client, CosConfig + store_address = deepcopy(storage_address) + store_address.pop('Bucket') + return CosS3Client(CosConfig(**store_address)) + diff --git a/python/fate_flow/manager/outputs/model/handel/__init__.py b/python/fate_flow/manager/outputs/model/handel/__init__.py new file mode 100644 index 000000000..bea35412a --- /dev/null +++ b/python/fate_flow/manager/outputs/model/handel/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_flow.manager.outputs.model.handel._base import IOHandle +from fate_flow.manager.outputs.model.handel._file import FileHandle +from fate_flow.manager.outputs.model.handel._mysql import MysqlHandel +from fate_flow.manager.outputs.model.handel._tencent_cos import TencentCosHandel + +__all__ = ["IOHandle", "FileHandle", "MysqlHandel", "TencentCosHandel"] diff --git a/python/fate_flow/manager/outputs/model/handel/_base.py b/python/fate_flow/manager/outputs/model/handel/_base.py new file mode 100644 index 000000000..a827efb04 --- /dev/null +++ b/python/fate_flow/manager/outputs/model/handel/_base.py @@ -0,0 +1,173 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import os.path +import tarfile +from typing import Union, List + +from ruamel import yaml +from werkzeug.datastructures import FileStorage + +from fate_flow.entity.spec.flow import Metadata +from fate_flow.errors.server_error import NoFoundModelOutput +from fate_flow.manager.outputs.model.model_meta import ModelMeta +from fate_flow.manager.operation.job_saver import JobSaver + + +class IOHandle(object): + @property + def name(self): + return self._name + + @staticmethod + def storage_key(model_id, model_version, role, party_id, task_name, output_key): + return os.path.join(model_id, model_version, role, party_id, task_name, output_key) + + @staticmethod + def parse_storage_key(storage_key): + return storage_key.split(os.sep) + + def download(self, job_id=None, model_id=None, model_version=None, role=None, party_id=None, task_name=None, + output_key=None): + model_metas = ModelMeta.query(model_id=model_id, model_version=model_version, task_name=task_name, + output_key=output_key, role=role, party_id=party_id, job_id=job_id) + if not model_metas: + raise ValueError("No found model") + return self._download(storage_key=model_metas[0].f_storage_key) + + def upload(self, model_file: FileStorage, job_id, task_name, output_key, model_id, model_version, role, + party_id, type_name): + storage_key = self.storage_key(model_id, model_version, role, party_id, task_name, output_key) + metas = self._upload(model_file=model_file, storage_key=storage_key) + self.log_meta(metas, storage_key, job_id=job_id, task_name=task_name, model_id=model_id, type_name=type_name, + model_version=model_version, output_key=output_key, role=role, party_id=party_id) + + def save_as(self, storage_key, temp_path): + os.makedirs(os.path.dirname(temp_path), exist_ok=True) + return self._save_as(storage_key, temp_path) + + def load(self, file, storage_key, model_id, model_version, role, party_id, task_name, output_key): + metas = self._load(file=file, storage_key=storage_key) + self.log_meta(metas, storage_key, model_id=model_id, model_version=model_version, role=role, party_id=party_id, + task_name=task_name, output_key=output_key) + + def delete(self, **kwargs): + model_metas = ModelMeta.query(storage_engine=self.name, **kwargs) + if not model_metas: + raise NoFoundModelOutput(**kwargs) + for meta in model_metas: + try: + self._delete(storage_key=meta.f_storage_key) + except: + pass + return self.delete_meta(storage_engine=self.name, **kwargs) + + def log_meta(self, model_metas, storage_key, model_id, model_version, output_key, task_name, role, party_id, + job_id="", type_name=""): + model_info = { + "storage_key": storage_key, + "storage_engine": self.name, + "model_id": model_id, + "model_version": model_version, + "job_id": job_id, + "role": role, + "party_id": party_id, + "task_name": task_name, + "output_key": output_key, + "meta_data": model_metas, + "type_name": type_name + } + ModelMeta.save(**model_info) + + @staticmethod + def delete_meta(**kwargs): + return ModelMeta.delete(**kwargs) + + def meta_info(self, model_meta: Metadata): + execution_id = model_meta.model_overview.party.party_task_id + task = JobSaver.query_task_by_execution_id(execution_id=execution_id) + job = JobSaver.query_job(job_id=task.f_job_id, role=task.f_role, party_id=task.f_party_id)[0] + _meta_info = { + "model_id": job.f_model_id, + "model_version": job.f_model_version, + "job_id": task.f_job_id, + "role": task.f_role, + "party_id": task.f_party_id, + "task_name": task.f_task_name, + "storage_engine": self.name + } + return _meta_info + + def read(self, job_id, role, party_id, task_name): + models = ModelMeta.query(job_id=job_id, role=role, party_id=party_id, task_name=task_name, reverse=True) + if not models: + raise NoFoundModelOutput(job_id=job_id, role=role, party_id=party_id, task_name=task_name) + model_dict = {} + for model in models: + model_dict[model.f_output_key] = self._read(model.f_storage_key, model.f_meta_data) + return model_dict + + @property + def _name(self): + raise NotImplementedError() + + def _upload(self, **kwargs): + raise NotImplementedError() + + def _download(self, **kwargs): + raise NotImplementedError() + + def _read(self, storage_key, metas): + raise NotImplementedError() + + def _delete(self, storage_key): + raise NotImplementedError() + + def _save_as(self, storage_key, path): + raise NotImplementedError() + + def _load(self, file, storage_key): + raise NotImplementedError() + + @classmethod + def read_meta(cls, _tar: tarfile.TarFile) -> Union[Metadata, List[Metadata]]: + meta_list = [] + for name in _tar.getnames(): + if name.endswith("yaml"): + fp = _tar.extractfile(name).read() + meta = yaml.safe_load(fp) + meta_list.append(meta) + return meta_list + + @classmethod + def read_model(cls, _tar: tarfile.TarFile, metas): + model_cache = {} + for _meta in metas: + meta = Metadata(**_meta) + try: + fp = _tar.extractfile(meta.model_key).read() + _json_model = json.loads(fp) + if meta.index is None: + return _json_model + model_cache[meta.index] = _json_model + except Exception as e: + pass + return [model_cache[_k] for _k in sorted(model_cache)] + + @staticmethod + def update_meta(): + pass + + diff --git a/python/fate_flow/manager/outputs/model/handel/_file.py b/python/fate_flow/manager/outputs/model/handel/_file.py new file mode 100644 index 000000000..4ab6317b2 --- /dev/null +++ b/python/fate_flow/manager/outputs/model/handel/_file.py @@ -0,0 +1,74 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io +import os.path +import shutil +import tarfile + +from flask import send_file +from werkzeug.datastructures import FileStorage + +from fate_flow.entity.spec.flow import FileStorageSpec +from fate_flow.entity.types import ModelStorageEngine +from fate_flow.manager.outputs.model.handel import IOHandle +from fate_flow.runtime.system_settings import MODEL_STORE_PATH + + +class FileHandle(IOHandle): + def __init__(self, engine_address: FileStorageSpec): + self.path = engine_address.path if engine_address.path else MODEL_STORE_PATH + + @property + def _name(self): + return ModelStorageEngine.FILE + + def _upload(self, model_file: FileStorage, storage_key): + _path = self._generate_model_storage_path(storage_key) + os.makedirs(os.path.dirname(_path), exist_ok=True) + model_file.save(_path) + model_metas = self.read_meta(self._tar_io(_path)) + return model_metas + + def _download(self, storage_key): + _p = self._generate_model_storage_path(storage_key) + return send_file(_p, download_name=os.path.basename(_p), as_attachment=True, mimetype='application/x-tar') + + def _save_as(self, storage_key, path): + _p = self._generate_model_storage_path(storage_key) + shutil.copy(_p, path) + return path + + def _load(self, file, storage_key): + _path = self._generate_model_storage_path(storage_key) + os.makedirs(os.path.dirname(_path), exist_ok=True) + shutil.copy(file, _path) + return self.read_meta(self._tar_io(_path)) + + def _read(self, storage_key, metas): + _p = self._generate_model_storage_path(storage_key) + _tar_io = self._tar_io(_p) + return self.read_model(_tar_io, metas) + + def _delete(self, storage_key): + _p = self._generate_model_storage_path(storage_key) + return os.remove(_p) + + @staticmethod + def _tar_io(path): + with open(path, "rb") as f: + return tarfile.open(fileobj=io.BytesIO(f.read())) + + def _generate_model_storage_path(self, storage_key): + return os.path.join(self.path, storage_key) diff --git a/python/fate_flow/manager/outputs/model/handel/_mysql.py b/python/fate_flow/manager/outputs/model/handel/_mysql.py new file mode 100644 index 000000000..1a52f8420 --- /dev/null +++ b/python/fate_flow/manager/outputs/model/handel/_mysql.py @@ -0,0 +1,72 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io +import tarfile + +from flask import send_file +from werkzeug.datastructures import FileStorage + +from fate_flow.entity.spec.flow import MysqlStorageSpec +from fate_flow.entity.types import ModelStorageEngine +from fate_flow.manager.outputs.model.engine import MysqlModelStorage +from fate_flow.manager.outputs.model.handel import IOHandle + + +class MysqlHandel(IOHandle): + def __init__(self, engine_address: MysqlStorageSpec, decrypt_key=None): + self.engine = MysqlModelStorage(engine_address.dict(), decrypt_key=decrypt_key) + + @property + def _name(self): + return ModelStorageEngine.MYSQL + + def _upload(self, model_file: FileStorage, storage_key): + memory = io.BytesIO() + model_file.save(memory) + metas = self.read_meta(self._tar_io(memory)) + self.engine.store(memory, storage_key) + return metas + + def _download(self, storage_key): + memory = self.engine.read(storage_key) + memory.seek(0) + return send_file(memory, download_name=storage_key, as_attachment=True, mimetype='application/x-tar') + + def _read(self, storage_key, metas): + memory = self.engine.read(storage_key) + _tar_io = self._tar_io(memory) + return self.read_model(_tar_io, metas) + + def _delete(self, storage_key): + self.engine.delete(storage_key=storage_key) + + def _load(self, file, storage_key): + with open(file, "rb") as memory: + memory.seek(0) + model_meta = self.read_meta(self._tar_io(memory)) + self.engine.store(memory, storage_key) + return model_meta + + def _save_as(self, storage_key, path): + memory = self.engine.read(storage_key) + memory.seek(0) + with open(path, "wb") as f: + f.write(memory.read()) + return path + + @staticmethod + def _tar_io(memory): + memory.seek(0) + return tarfile.open(fileobj=memory) diff --git a/python/fate_flow/manager/outputs/model/handel/_tencent_cos.py b/python/fate_flow/manager/outputs/model/handel/_tencent_cos.py new file mode 100644 index 000000000..eb9424317 --- /dev/null +++ b/python/fate_flow/manager/outputs/model/handel/_tencent_cos.py @@ -0,0 +1,71 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io +import tarfile + +from flask import send_file +from werkzeug.datastructures import FileStorage + +from fate_flow.entity.spec.flow import TencentCosStorageSpec +from fate_flow.entity.types import ModelStorageEngine +from fate_flow.manager.outputs.model.engine import TencentCosStorage +from fate_flow.manager.outputs.model.handel import IOHandle + + +class TencentCosHandel(IOHandle): + def __init__(self, engine_address: TencentCosStorageSpec, decrypt_key: str = None): + self.engine = TencentCosStorage(engine_address.dict(), decrypt_key) + + @property + def _name(self): + return ModelStorageEngine.TENCENT_COS + + def _upload(self, model_file: FileStorage, storage_key): + memory = io.BytesIO() + model_file.save(memory) + metas = self.read_meta(self._tar_io(memory)) + self.engine.store(memory, storage_key) + return metas + + def _download(self, storage_key): + memory = self.engine.read(storage_key) + memory.seek(0) + return send_file(memory, as_attachment=True, download_name=storage_key, mimetype='application/x-tar') + + def _read(self, storage_key, metas): + memory = self.engine.read(storage_key) + _tar_io = self._tar_io(memory) + return self.read_model(_tar_io, metas) + + def _delete(self, storage_key): + return self.engine.delete(storage_key=storage_key) + + def _load(self, file, storage_key): + with open(file, "rb") as memory: + model_meta = self.read_meta(self._tar_io(memory)) + self.engine.store(memory, storage_key) + return model_meta + + def _save_as(self, storage_key, path): + memory = self.engine.read(storage_key) + memory.seek(0) + with open(path, "wb") as f: + f.write(memory.read()) + return path + + @staticmethod + def _tar_io(memory): + memory.seek(0) + return tarfile.open(fileobj=memory) diff --git a/python/fate_flow/manager/outputs/model/model_manager.py b/python/fate_flow/manager/outputs/model/model_manager.py new file mode 100644 index 000000000..18e935b2b --- /dev/null +++ b/python/fate_flow/manager/outputs/model/model_manager.py @@ -0,0 +1,93 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import shutil +from tempfile import TemporaryDirectory + +from werkzeug.datastructures import FileStorage + +from fate_flow.entity.spec.flow import FileStorageSpec, MysqlStorageSpec, TencentCosStorageSpec +from fate_flow.entity.types import ModelStorageEngine +from fate_flow.manager.outputs.model.handel import FileHandle, MysqlHandel, TencentCosHandel +from fate_flow.manager.outputs.model.model_meta import ModelMeta +from fate_flow.runtime.system_settings import MODEL_STORE +from fate_flow.errors.server_error import NoFoundModelOutput + + +class PipelinedModel(object): + engine = MODEL_STORE.get("engine") + decrypt_key = MODEL_STORE.get("decrypt_key") + if engine == ModelStorageEngine.FILE: + handle = FileHandle(engine_address=FileStorageSpec(**MODEL_STORE.get(engine))) + elif engine == ModelStorageEngine.MYSQL: + handle = MysqlHandel(engine_address=MysqlStorageSpec(**MODEL_STORE.get(engine)), decrypt_key=decrypt_key) + elif engine == ModelStorageEngine.TENCENT_COS: + handle = TencentCosHandel(engine_address=TencentCosStorageSpec(**MODEL_STORE.get(engine)), + decrypt_key=decrypt_key) + else: + raise ValueError(f"Model storage engine {engine} is not supported.") + + @classmethod + def upload_model(cls, model_file: FileStorage, job_id: str, task_name, output_key, model_id, model_version, role, + party_id, type_name): + return cls.handle.upload(model_file, job_id, task_name, output_key, model_id, model_version, role, + party_id, type_name) + + @classmethod + def download_model(cls, **kwargs): + return cls.handle.download(**kwargs) + + @classmethod + def read_model(cls, job_id, role, party_id, task_name): + return cls.handle.read(job_id, role, party_id, task_name) + + @classmethod + def delete_model(cls, **kwargs): + return cls.handle.delete(**kwargs) + + @classmethod + def export_model(cls, model_id, model_version, role, party_id, dir_path): + _key_list = cls.get_model_storage_key(model_id=model_id, model_version=model_version, role=role, party_id=party_id) + if not _key_list: + raise NoFoundModelOutput(model_id=model_id, model_version=model_version, role=role, party_id=party_id) + with TemporaryDirectory() as temp_dir: + for _k in _key_list: + temp_path = os.path.join(temp_dir, _k) + cls.handle.save_as(storage_key=_k, temp_path=temp_path) + os.makedirs(dir_path, exist_ok=True) + shutil.make_archive(os.path.join(dir_path, f"{model_id}_{model_version}_{role}_{party_id}"), 'zip', temp_dir) + + @classmethod + def import_model(cls, model_id, model_version, path, temp_dir): + base_dir = os.path.dirname(path) + shutil.unpack_archive(path, base_dir, 'zip') + for dirpath, dirnames, filenames in os.walk(base_dir): + for filename in filenames: + model_path = os.path.join(dirpath, filename) + # exclude original model packs + if model_path != path: + _storage_key = model_path.lstrip(f"{temp_dir}{os.sep}") + _, _, role, party_id, task_name, output_key = cls.handle.parse_storage_key(_storage_key) + storage_key = cls.handle.storage_key(model_id, model_version, role, party_id, task_name, output_key) + cls.handle.load(model_path, storage_key, model_id, model_version, role=role, party_id=party_id, + task_name=task_name, output_key=output_key) + + @classmethod + def get_model_storage_key(cls, **kwargs): + _key_list = [] + _model_metas = ModelMeta.query(**kwargs) + for _meta in _model_metas: + _key_list.append(_meta.f_storage_key) + return _key_list diff --git a/python/fate_flow/manager/outputs/model/model_meta.py b/python/fate_flow/manager/outputs/model/model_meta.py new file mode 100644 index 000000000..696aca1e2 --- /dev/null +++ b/python/fate_flow/manager/outputs/model/model_meta.py @@ -0,0 +1,33 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_flow.db.base_models import BaseModelOperate +from fate_flow.db.db_models import PipelineModelMeta +from fate_flow.utils.wraps_utils import filter_parameters + + +class ModelMeta(BaseModelOperate): + @classmethod + def save(cls, **meta_info): + cls._create_entity(PipelineModelMeta, meta_info) + + @classmethod + @filter_parameters() + def query(cls, **kwargs): + return cls._query(PipelineModelMeta, **kwargs) + + @classmethod + @filter_parameters() + def delete(cls, **kwargs): + return cls._delete(PipelineModelMeta, **kwargs) diff --git a/python/fate_flow/manager/permission/__init__.py b/python/fate_flow/manager/permission/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/manager/pipeline/__init__.py b/python/fate_flow/manager/pipeline/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/manager/pipeline/pipeline.py b/python/fate_flow/manager/pipeline/pipeline.py new file mode 100644 index 000000000..20d192a24 --- /dev/null +++ b/python/fate_flow/manager/pipeline/pipeline.py @@ -0,0 +1,60 @@ +from fate_flow.manager.operation.job_saver import JobSaver +from fate_flow.entity.spec.dag import DAGSchema +from fate_flow.controller.parser import JobParser +from fate_flow.utils import job_utils + + +def pipeline_dag_dependency(job): + component_list = [] + component_module, dependence_dict, component_stage = {}, {}, {} + job_parser = JobParser(DAGSchema(**job.f_dag)) + task_list = job_parser.party_topological_sort(role=job.f_role, party_id=job.f_party_id) + for task_name in task_list: + task_node = job_parser.get_task_node(role=job.f_role, party_id=job.f_party_id, task_name=task_name) + parties = job_parser.get_task_runtime_parties(task_name=task_name) + need_run = job_utils.check_party_in(job.f_role, job.f_party_id, parties) + if not need_run: + continue + component_list.append(task_name) + component_module[task_name] = task_node.component_ref + dependence_dict[task_name] = [] + + component_stage[task_name] = task_node.stage + upstream_inputs = task_node.upstream_inputs + model_type_list = list(upstream_inputs.keys()) + for model_type in model_type_list: + for data_type in list(upstream_inputs[model_type].keys()): + data_value = upstream_inputs[model_type][data_type] + if isinstance(data_value, list): + for value in data_value: + up_output_info = [value.output_artifact_key] + if task_name == value.producer_task: + continue + dependence_dict[task_name].append({ + "type": data_type, + "model_type": model_type, + "component_name": value.producer_task if up_output_info else None, + "up_output_info": up_output_info + }) + else: + up_output_info = [data_value.output_artifact_key] + + if task_name == data_value.producer_task: + continue + dependence_dict[task_name].append({ + "type": data_type, + "model_type": model_type, + "component_name": data_value.producer_task if up_output_info else None, + "up_output_info": up_output_info + }) + if not model_type_list: + dependence_dict[task_name].append({"up_output_info": []}) + + base_dependency = { + "component_list": component_list, + "component_stage": component_stage, + "component_module": component_module, + "dependencies": dependence_dict + } + + return base_dependency diff --git a/python/fate_flow/manager/pipeline_manager.py b/python/fate_flow/manager/pipeline_manager.py deleted file mode 100644 index df9e08670..000000000 --- a/python/fate_flow/manager/pipeline_manager.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_flow.settings import stat_logger -from fate_flow.utils import detect_utils, schedule_utils -from fate_flow.operation.job_saver import JobSaver - - -def pipeline_dag_dependency(job_info): - try: - detect_utils.check_config(job_info, required_arguments=["party_id", "role"]) - component_need_run = {} - if job_info.get('job_id'): - jobs = JobSaver.query_job(job_id=job_info["job_id"], party_id=job_info["party_id"], role=job_info["role"]) - if not jobs: - raise Exception('query job {} failed'.format(job_info.get('job_id', ''))) - job = jobs[0] - dsl_parser = schedule_utils.get_job_dsl_parser(dsl=job.f_dsl, - runtime_conf=job.f_runtime_conf_on_party, - train_runtime_conf=job.f_train_runtime_conf) - tasks = JobSaver.query_task(job_id=job_info["job_id"], party_id=job_info["party_id"], role=job_info["role"], only_latest=True) - for task in tasks: - need_run = task.f_component_parameters.get("ComponentParam", {}).get("need_run", True) - component_need_run[task.f_component_name] = need_run - else: - dsl_parser = schedule_utils.get_job_dsl_parser(dsl=job_info.get('job_dsl', {}), - runtime_conf=job_info.get('job_runtime_conf', {}), - train_runtime_conf=job_info.get('job_train_runtime_conf', {})) - dependency = dsl_parser.get_dependency() - dependency["component_need_run"] = component_need_run - return dependency - except Exception as e: - stat_logger.exception(e) - raise e diff --git a/python/fate_flow/manager/provider_manager.py b/python/fate_flow/manager/provider_manager.py deleted file mode 100644 index df5b265c4..000000000 --- a/python/fate_flow/manager/provider_manager.py +++ /dev/null @@ -1,172 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os.path -import sys -from copy import deepcopy - -from fate_arch.common import file_utils -from fate_arch.common.versions import get_versions -from fate_flow.controller.version_controller import VersionController -from fate_flow.entity import ComponentProvider -from fate_flow.db.component_registry import ComponentRegistry -from fate_flow.db.job_default_config import JobDefaultConfig -from fate_flow.manager.worker_manager import WorkerManager -from fate_flow.entity.types import WorkerName -from fate_flow.settings import stat_logger -from fate_flow.utils.base_utils import get_fate_flow_python_directory - - -class ProviderManager: - @classmethod - def register_default_providers(cls): - code, result = cls.register_fate_flow_provider() - if code != 0: - raise Exception(f"register fate flow tools component failed") - code, result, provider = cls.register_default_fate_provider() - if code != 0: - raise Exception(f"register default fate algorithm component failed") - return provider - - @classmethod - def register_fate_flow_provider(cls): - provider = cls.get_fate_flow_provider() - return WorkerManager.start_general_worker(worker_name=WorkerName.PROVIDER_REGISTRAR, provider=provider, run_in_subprocess=False) - - @classmethod - def register_default_fate_provider(cls): - provider = cls.get_default_fate_provider() - sys.path.append(provider.env["PYTHONPATH"]) - code, result = WorkerManager.start_general_worker(worker_name=WorkerName.PROVIDER_REGISTRAR, provider=provider, run_in_subprocess=False) - return code, result, provider - - @classmethod - def get_fate_flow_provider(cls): - path = get_fate_flow_python_directory("fate_flow") - provider = ComponentProvider(name="fate_flow", version=get_versions()["FATEFlow"], path=path, class_path=ComponentRegistry.get_default_class_path()) - return provider - - @classmethod - def get_default_fate_provider_env(cls): - provider = cls.get_default_fate_provider() - return provider.env - - @classmethod - def get_default_fate_provider(cls): - path = JobDefaultConfig.default_component_provider_path.split("/") - path = file_utils.get_fate_python_directory(*path) - if not os.path.exists(path): - raise Exception(f"default fate provider not exists: {path}") - provider = ComponentProvider(name="fate", version=get_versions()["FATE"], path=path, class_path=ComponentRegistry.get_default_class_path()) - return provider - - @classmethod - def if_default_provider(cls, provider: ComponentProvider): - if provider == cls.get_fate_flow_provider() or provider == cls.get_default_fate_provider(): - return True - else: - return False - - @classmethod - def fill_fate_flow_provider(cls, dsl): - dest_dsl = deepcopy(dsl) - fate_flow_provider = cls.get_fate_flow_provider() - support_components = ComponentRegistry.get_provider_components(fate_flow_provider.name, fate_flow_provider.version) - provider_key = f"{fate_flow_provider.name}@{fate_flow_provider.version}" - for cpn, config in dsl["components"].items(): - if config["module"] in support_components: - dest_dsl["components"][cpn]["provider"] = provider_key - return dest_dsl - - @classmethod - def get_fate_flow_component_module(cls): - fate_flow_provider = cls.get_fate_flow_provider() - return ComponentRegistry.get_provider_components(fate_flow_provider.name, fate_flow_provider.version) - - @classmethod - def get_provider_object(cls, provider_info, check_registration=True): - name, version = provider_info["name"], provider_info["version"] - if check_registration and ComponentRegistry.get_providers().get(name, {}).get(version, None) is None: - raise Exception(f"{name} {version} provider is not registered") - path = ComponentRegistry.get_providers().get(name, {}).get(version, {}).get("path", []) - class_path = ComponentRegistry.get_providers().get(name, {}).get(version, {}).get("class_path", None) - if class_path is None: - class_path = ComponentRegistry.REGISTRY["default_settings"]["class_path"] - return ComponentProvider(name=name, version=version, path=path, class_path=class_path) - - @classmethod - def get_job_provider_group(cls, dsl_parser, role, party_id, components: list = None, check_registration=True, - runtime_conf=None, check_version=False, is_scheduler=False): - if is_scheduler: - # local provider - providers_info = dsl_parser.get_job_providers(provider_detail=ComponentRegistry.REGISTRY) - else: - providers_info = dsl_parser.get_job_providers(provider_detail=ComponentRegistry.REGISTRY, conf=runtime_conf, - local_role=role, local_party_id=party_id) - if check_version: - VersionController.job_provider_version_check(providers_info, local_role=role, local_party_id=party_id) - group = {} - if role in providers_info and not is_scheduler: - providers_info = providers_info.get(role, {}).get(int(party_id), {}) or\ - providers_info.get(role, {}).get(str(party_id), {}) - if components is not None: - _providers_info = {} - for component_name in components: - _providers_info[component_name] = providers_info.get(component_name) - providers_info = _providers_info - for component_name, provider_info in providers_info.items(): - provider = cls.get_provider_object(provider_info["provider"], check_registration=check_registration) - group_key = "@".join([provider.name, provider.version]) - if group_key not in group: - group[group_key] = { - "provider": provider.to_dict(), - "if_default_provider": cls.if_default_provider(provider), - "components": [component_name] - } - else: - group[group_key]["components"].append(component_name) - return group - - @classmethod - def get_component_provider(cls, dsl_parser, component_name): - providers = dsl_parser.get_job_providers(provider_detail=ComponentRegistry.REGISTRY) - return cls.get_provider_object(providers[component_name]["provider"]) - - @classmethod - def get_component_parameters(cls, dsl_parser, component_name, role, party_id, provider: ComponentProvider = None, previous_components_parameters: dict = None): - if not provider: - provider = cls.get_component_provider(dsl_parser=dsl_parser, - component_name=component_name) - parameters = dsl_parser.parse_component_parameters(component_name, - ComponentRegistry.REGISTRY, - provider.name, - provider.version, - local_role=role, - local_party_id=int(party_id), - previous_parameters=previous_components_parameters) - user_specified_parameters = dsl_parser.parse_user_specified_component_parameters(component_name, - ComponentRegistry.REGISTRY, - provider.name, - provider.version, - local_role=role, - local_party_id=int(party_id), - previous_parameters=previous_components_parameters) - return parameters, user_specified_parameters - - @classmethod - def get_component_run_info(cls, dsl_parser, component_name, role, party_id, previous_components_parameters: dict = None): - provider = cls.get_component_provider(dsl_parser, component_name) - parameters, user_specified_parameters = cls.get_component_parameters(dsl_parser, component_name, role, party_id, provider, previous_components_parameters) - return provider, parameters, user_specified_parameters diff --git a/python/fate_flow/manager/resource_manager.py b/python/fate_flow/manager/resource_manager.py deleted file mode 100644 index f6a5e65fe..000000000 --- a/python/fate_flow/manager/resource_manager.py +++ /dev/null @@ -1,356 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import math -import typing - -from fate_arch.common import EngineType -from fate_arch.common import base_utils -from fate_flow.utils.log_utils import schedule_logger -from fate_arch.computing import ComputingEngine -from fate_arch.common import engine_utils -from fate_flow.db.db_models import DB, EngineRegistry, Job -from fate_flow.entity.types import ResourceOperation -from fate_flow.entity import RunParameters -from fate_flow.operation.job_saver import JobSaver -from fate_flow.settings import stat_logger, IGNORE_RESOURCE_ROLES, SUPPORT_IGNORE_RESOURCE_ENGINES, \ - IGNORE_RESOURCE_COMPUTING_ENGINE, ENGINES -from fate_flow.utils import job_utils -from fate_flow.db.job_default_config import JobDefaultConfig - - -class ResourceManager(object): - @classmethod - def initialize(cls): - engines_config, engine_group_map = engine_utils.get_engines_config_from_conf(group_map=True) - for engine_type, engine_configs in engines_config.items(): - for engine_name, engine_config in engine_configs.items(): - cls.register_engine(engine_type=engine_type, engine_name=engine_name, engine_entrance=engine_group_map[engine_type][engine_name], engine_config=engine_config) - - @classmethod - @DB.connection_context() - def register_engine(cls, engine_type, engine_name, engine_entrance, engine_config): - nodes = engine_config.get("nodes", 1) - cores = engine_config.get("cores_per_node", 0) * nodes * JobDefaultConfig.total_cores_overweight_percent - memory = engine_config.get("memory_per_node", 0) * nodes * JobDefaultConfig.total_memory_overweight_percent - filters = [EngineRegistry.f_engine_type == engine_type, EngineRegistry.f_engine_name == engine_name] - resources = EngineRegistry.select().where(*filters) - if resources: - resource = resources[0] - update_fields = {} - update_fields[EngineRegistry.f_engine_config] = engine_config - update_fields[EngineRegistry.f_cores] = cores - update_fields[EngineRegistry.f_memory] = memory - update_fields[EngineRegistry.f_remaining_cores] = EngineRegistry.f_remaining_cores + ( - cores - resource.f_cores) - update_fields[EngineRegistry.f_remaining_memory] = EngineRegistry.f_remaining_memory + ( - memory - resource.f_memory) - update_fields[EngineRegistry.f_nodes] = nodes - operate = EngineRegistry.update(update_fields).where(*filters) - update_status = operate.execute() > 0 - if update_status: - stat_logger.info(f"update {engine_type} engine {engine_name} {engine_entrance} registration information") - else: - stat_logger.info(f"update {engine_type} engine {engine_name} {engine_entrance} registration information takes no effect") - else: - resource = EngineRegistry() - resource.f_create_time = base_utils.current_timestamp() - resource.f_engine_type = engine_type - resource.f_engine_name = engine_name - resource.f_engine_entrance = engine_entrance - resource.f_engine_config = engine_config - - resource.f_cores = cores - resource.f_memory = memory - resource.f_remaining_cores = cores - resource.f_remaining_memory = memory - resource.f_nodes = nodes - try: - resource.save(force_insert=True) - except Exception as e: - stat_logger.warning(e) - stat_logger.info(f"create {engine_type} engine {engine_name} {engine_entrance} registration information") - - @classmethod - def check_resource_apply(cls, job_parameters: RunParameters, role, party_id, engines_info): - computing_engine, cores, memory = cls.calculate_job_resource(job_parameters=job_parameters, role=role, party_id=party_id) - max_cores_per_job = math.floor(engines_info[EngineType.COMPUTING].f_cores * JobDefaultConfig.max_cores_percent_per_job) \ - if engines_info.get(EngineType.COMPUTING) else 0 - - if cores > max_cores_per_job: - return False, cores, max_cores_per_job - return True, cores, max_cores_per_job - - @classmethod - def apply_for_job_resource(cls, job_id, role, party_id): - return cls.resource_for_job(job_id=job_id, role=role, party_id=party_id, operation_type=ResourceOperation.APPLY) - - @classmethod - def return_job_resource(cls, job_id, role, party_id): - return cls.resource_for_job(job_id=job_id, role=role, party_id=party_id, - operation_type=ResourceOperation.RETURN) - - @classmethod - def query_resource(cls, resource_in_use=True, engine_name=None): - if not engine_name: - engine_name = ENGINES.get(EngineType.COMPUTING) - use_resource_jobs = JobSaver.query_job(resource_in_use=resource_in_use) - used = [] - for job in use_resource_jobs: - used.append({"job_id": job.f_job_id, "role": job.f_role, "party_id": job.f_party_id, - "core": job.f_cores, "memory": job.f_memory}) - computing_engine_resource = cls.get_engine_registration_info(engine_type=EngineType.COMPUTING, engine_name=engine_name) - return used, computing_engine_resource.to_dict() if computing_engine_resource else {} - - @classmethod - def return_resource(cls, job_id): - jobs = JobSaver.query_job(job_id=job_id) - if not jobs: - raise Exception(f'no found job {job_id}') - return_resource_job_list = [] - for job in jobs: - job_info = {"job_id": job.f_job_id, "role": job.f_role, "party_id": job.f_party_id, - "resource_in_use": job.f_resource_in_use, "resource_return_status": False} - if job.f_resource_in_use: - return_status = cls.return_job_resource(job.f_job_id, job.f_role, job.f_party_id) - job_info["resource_return_status"] = return_status - return_resource_job_list.append(job_info) - return return_resource_job_list - - @classmethod - @DB.connection_context() - def resource_for_job(cls, job_id, role, party_id, operation_type: ResourceOperation): - operate_status = False - engine_name, cores, memory = cls.calculate_job_resource(job_id=job_id, role=role, party_id=party_id) - try: - with DB.atomic(): - updates = { - Job.f_engine_type: EngineType.COMPUTING, - Job.f_engine_name: engine_name, - Job.f_cores: cores, - Job.f_memory: memory, - } - filters = [ - Job.f_job_id == job_id, - Job.f_role == role, - Job.f_party_id == party_id, - ] - if operation_type is ResourceOperation.APPLY: - updates[Job.f_remaining_cores] = cores - updates[Job.f_remaining_memory] = memory - updates[Job.f_resource_in_use] = True - updates[Job.f_apply_resource_time] = base_utils.current_timestamp() - filters.append(Job.f_resource_in_use == False) - elif operation_type is ResourceOperation.RETURN: - updates[Job.f_resource_in_use] = False - updates[Job.f_return_resource_time] = base_utils.current_timestamp() - filters.append(Job.f_resource_in_use == True) - operate = Job.update(updates).where(*filters) - record_status = operate.execute() > 0 - if not record_status: - raise RuntimeError(f"record job {job_id} resource {operation_type} failed on {role} {party_id}") - - if cores or memory: - filters, updates = cls.update_resource_sql(resource_model=EngineRegistry, - cores=cores, - memory=memory, - operation_type=operation_type, - ) - filters.append(EngineRegistry.f_engine_type == EngineType.COMPUTING) - filters.append(EngineRegistry.f_engine_name == engine_name) - operate = EngineRegistry.update(updates).where(*filters) - apply_status = operate.execute() > 0 - else: - apply_status = True - if not apply_status: - raise RuntimeError( - f"update engine {engine_name} record for job {job_id} resource {operation_type} on {role} {party_id} failed") - operate_status = True - except Exception as e: - schedule_logger(job_id).warning(e) - schedule_logger(job_id).warning( - f"{operation_type} job resource(cores {cores} memory {memory}) on {role} {party_id} failed") - operate_status = False - finally: - remaining_cores, remaining_memory = cls.get_remaining_resource(EngineRegistry, - [ - EngineRegistry.f_engine_type == EngineType.COMPUTING, - EngineRegistry.f_engine_name == engine_name]) - operate_msg = "successfully" if operate_status else "failed" - schedule_logger(job_id).info( - f"{operation_type} job resource(cores {cores} memory {memory}) on {role} {party_id} {operate_msg}, remaining cores: {remaining_cores} remaining memory: {remaining_memory}") - return operate_status - - @classmethod - def adapt_engine_parameters(cls, role, job_parameters: RunParameters, create_initiator_baseline=False): - computing_engine_info = ResourceManager.get_engine_registration_info(engine_type=EngineType.COMPUTING, - engine_name=job_parameters.computing_engine) - if not job_parameters.adaptation_parameters or create_initiator_baseline: - job_parameters.adaptation_parameters = { - "task_nodes": 0, - "task_cores_per_node": 0, - "task_memory_per_node": 0, - # request_task_cores base on initiator and distribute to all parties, using job conf parameters or initiator fateflow server default settings - "request_task_cores": int(job_parameters.task_cores) if job_parameters.task_cores else JobDefaultConfig.task_cores, - "if_initiator_baseline": True - } - else: - # use initiator baseline - if role == "arbiter": - job_parameters.adaptation_parameters["request_task_cores"] = 1 - elif "request_task_cores" not in job_parameters.adaptation_parameters: - # compatibility 1.5.0 - job_parameters.adaptation_parameters["request_task_cores"] = job_parameters.adaptation_parameters["task_nodes"] * job_parameters.adaptation_parameters["task_cores_per_node"] - - job_parameters.adaptation_parameters["if_initiator_baseline"] = False - adaptation_parameters = job_parameters.adaptation_parameters - - if job_parameters.computing_engine in {ComputingEngine.STANDALONE, ComputingEngine.EGGROLL}: - adaptation_parameters["task_nodes"] = computing_engine_info.f_nodes - if int(job_parameters.eggroll_run.get("eggroll.session.processors.per.node", 0)) > 0: - adaptation_parameters["task_cores_per_node"] = int(job_parameters.eggroll_run["eggroll.session.processors.per.node"]) - else: - adaptation_parameters["task_cores_per_node"] = max(1, int(adaptation_parameters["request_task_cores"] / adaptation_parameters["task_nodes"])) - if not create_initiator_baseline: - # set the adaptation parameters to the actual engine operation parameters - job_parameters.eggroll_run["eggroll.session.processors.per.node"] = adaptation_parameters["task_cores_per_node"] - elif job_parameters.computing_engine == ComputingEngine.SPARK or job_parameters.computing_engine == ComputingEngine.LINKIS_SPARK: - adaptation_parameters["task_nodes"] = int(job_parameters.spark_run.get("num-executors", computing_engine_info.f_nodes)) - if int(job_parameters.spark_run.get("executor-cores", 0)) > 0: - adaptation_parameters["task_cores_per_node"] = int(job_parameters.spark_run["executor-cores"]) - else: - adaptation_parameters["task_cores_per_node"] = max(1, int(adaptation_parameters["request_task_cores"] / adaptation_parameters["task_nodes"])) - if not create_initiator_baseline: - # set the adaptation parameters to the actual engine operation parameters - job_parameters.spark_run["num-executors"] = adaptation_parameters["task_nodes"] - job_parameters.spark_run["executor-cores"] = adaptation_parameters["task_cores_per_node"] - - @classmethod - def calculate_job_resource(cls, job_parameters: RunParameters = None, job_id=None, role=None, party_id=None): - if not job_parameters: - job_parameters = job_utils.get_job_parameters(job_id=job_id, - role=role, - party_id=party_id) - job_parameters = RunParameters(**job_parameters) - - cores = 0 - memory = 0 - - if not (job_parameters.computing_engine in IGNORE_RESOURCE_COMPUTING_ENGINE or - role in IGNORE_RESOURCE_ROLES and job_parameters.computing_engine in SUPPORT_IGNORE_RESOURCE_ENGINES): - cores = (int(job_parameters.adaptation_parameters["task_cores_per_node"] or 0) * - int(job_parameters.adaptation_parameters["task_nodes"] or 0) * - int(job_parameters.task_parallelism or 0)) - memory = (int(job_parameters.adaptation_parameters["task_memory_per_node"] or 0) * - int(job_parameters.adaptation_parameters["task_nodes"] or 0) * - int(job_parameters.task_parallelism or 0)) - - return job_parameters.computing_engine, cores, memory - - @classmethod - def calculate_task_resource(cls, task_parameters: RunParameters = None, task_info: dict = None): - if not task_parameters: - job_parameters = job_utils.get_job_parameters(job_id=task_info["job_id"], - role=task_info["role"], - party_id=task_info["party_id"]) - task_parameters = RunParameters(**job_parameters) - if task_parameters.computing_engine in IGNORE_RESOURCE_COMPUTING_ENGINE: - cores_per_task = 0 - memory_per_task = 0 - elif task_info["role"] in IGNORE_RESOURCE_ROLES and task_parameters.computing_engine in SUPPORT_IGNORE_RESOURCE_ENGINES: - cores_per_task = 0 - memory_per_task = 0 - else: - cores_per_task = task_parameters.adaptation_parameters["task_cores_per_node"] * \ - task_parameters.adaptation_parameters["task_nodes"] - memory_per_task = task_parameters.adaptation_parameters["task_memory_per_node"] * \ - task_parameters.adaptation_parameters["task_nodes"] - return cores_per_task, memory_per_task - - @classmethod - def apply_for_task_resource(cls, task_info): - return ResourceManager.resource_for_task(task_info=task_info, operation_type=ResourceOperation.APPLY) - - @classmethod - def return_task_resource(cls, task_info): - return ResourceManager.resource_for_task(task_info=task_info, operation_type=ResourceOperation.RETURN) - - @classmethod - @DB.connection_context() - def resource_for_task(cls, task_info, operation_type): - cores_per_task, memory_per_task = cls.calculate_task_resource(task_info=task_info) - schedule_logger(task_info["job_id"]).info(f"cores_per_task:{cores_per_task}, memory_per_task:{memory_per_task}") - if cores_per_task or memory_per_task: - filters, updates = cls.update_resource_sql(resource_model=Job, - cores=cores_per_task, - memory=memory_per_task, - operation_type=operation_type, - ) - filters.append(Job.f_job_id == task_info["job_id"]) - filters.append(Job.f_role == task_info["role"]) - filters.append(Job.f_party_id == task_info["party_id"]) - filters.append(Job.f_resource_in_use == True) - operate = Job.update(updates).where(*filters) - operate_status = operate.execute() > 0 - else: - operate_status = True - if operate_status: - schedule_logger(task_info["job_id"]).info( - "task {} {} {} resource successfully".format(task_info["task_id"], - task_info["task_version"], operation_type)) - else: - schedule_logger(task_info["job_id"]).warning( - "task {} {} {} resource failed".format(task_info["task_id"], - task_info["task_version"], operation_type)) - return operate_status - - @classmethod - def update_resource_sql(cls, resource_model: typing.Union[EngineRegistry, Job], cores, memory, operation_type: ResourceOperation): - if operation_type is ResourceOperation.APPLY: - filters = [ - resource_model.f_remaining_cores >= cores, - resource_model.f_remaining_memory >= memory - ] - updates = {resource_model.f_remaining_cores: resource_model.f_remaining_cores - cores, - resource_model.f_remaining_memory: resource_model.f_remaining_memory - memory} - elif operation_type is ResourceOperation.RETURN: - filters = [] - updates = {resource_model.f_remaining_cores: resource_model.f_remaining_cores + cores, - resource_model.f_remaining_memory: resource_model.f_remaining_memory + memory} - else: - raise RuntimeError(f"can not support {operation_type} resource operation type") - return filters, updates - - @classmethod - @DB.connection_context() - def get_remaining_resource(cls, resource_model: typing.Union[EngineRegistry, Job], filters): - remaining_cores, remaining_memory = None, None - try: - objs = resource_model.select(resource_model.f_remaining_cores, resource_model.f_remaining_memory).where( - *filters) - if objs: - remaining_cores, remaining_memory = objs[0].f_remaining_cores, objs[0].f_remaining_memory - except Exception as e: - schedule_logger().exception(e) - finally: - return remaining_cores, remaining_memory - - @classmethod - @DB.connection_context() - def get_engine_registration_info(cls, engine_type, engine_name) -> EngineRegistry: - engines = EngineRegistry.select().where(EngineRegistry.f_engine_type == engine_type, - EngineRegistry.f_engine_name == engine_name) - if engines: - return engines[0] diff --git a/python/fate_flow/manager/service/__init__.py b/python/fate_flow/manager/service/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/fate_flow/manager/service/app_manager.py b/python/fate_flow/manager/service/app_manager.py new file mode 100644 index 000000000..8ecefac03 --- /dev/null +++ b/python/fate_flow/manager/service/app_manager.py @@ -0,0 +1,137 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from functools import wraps + +from fate_flow.db.base_models import BaseModelOperate +from fate_flow.db.permission_models import AppInfo, PartnerAppInfo +from fate_flow.entity.types import AppType +from fate_flow.errors.server_error import NoFoundAppid, RoleTypeError +from fate_flow.runtime.system_settings import ADMIN_KEY, CLIENT_AUTHENTICATION, APP_TOKEN_LENGTH, SITE_AUTHENTICATION, \ + PARTY_ID +from fate_flow.utils.base_utils import generate_random_id +from fate_flow.utils.wraps_utils import filter_parameters, switch_function, check_permission + + +class AppManager(BaseModelOperate): + @classmethod + def init(cls): + if CLIENT_AUTHENTICATION or SITE_AUTHENTICATION: + if cls.query_app(app_name="admin", init=True): + cls._delete(AppInfo, app_name="admin") + cls.create_app(app_name="admin", app_id="admin", app_token=ADMIN_KEY, app_type="admin", init=True) + app_info = cls.create_app(app_name=PARTY_ID, app_id=PARTY_ID, app_type=AppType.SITE, init=True) + if app_info: + cls.create_partner_app(party_id=PARTY_ID, app_id=app_info.get("app_id"), + app_token=app_info.get("app_token")) + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @check_permission(operate="create", types="client") + def create_app(cls, app_type, app_name, app_id=None, app_token=None, init=True): + if not app_id: + app_id = cls.generate_app_id() + if not app_token: + app_token = cls.generate_app_token() + app_info = { + "app_name": app_name, + "app_id": app_id, + "app_token": app_token, + "app_type": app_type + } + status = cls._create_entity(AppInfo, app_info) + if status: + return app_info + else: + return {} + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + def create_partner_app(cls, party_id, app_id=None, app_token=None): + app_info = { + "party_id": party_id, + "app_id": app_id, + "app_token": app_token, + } + status = cls._create_entity(PartnerAppInfo, app_info) + if status: + return app_info + else: + return {} + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @filter_parameters() + @check_permission(operate="delete", types="client") + def delete_app(cls, init=False, **kwargs): + return cls._delete(AppInfo, **kwargs) + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @filter_parameters() + def delete_partner_app(cls, init=False, **kwargs): + return cls._delete(PartnerAppInfo, **kwargs) + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @filter_parameters() + @check_permission(operate="query", types="client") + def query_app(cls, init=False, **kwargs): + return cls._query(AppInfo, **kwargs) + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + @filter_parameters() + def query_partner_app(cls, **kwargs): + return cls._query(PartnerAppInfo, **kwargs) + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + def generate_app_id(cls, length=8): + app_id = generate_random_id(length=length, only_number=True) + if cls.query_app(app_id=app_id): + cls.generate_app_id() + else: + return app_id + + @classmethod + @switch_function(CLIENT_AUTHENTICATION or SITE_AUTHENTICATION) + def generate_app_token(cls, length=APP_TOKEN_LENGTH): + return generate_random_id(length=length) + + @staticmethod + def check_app_id(func): + @wraps(func) + def _wrapper(*args, **kwargs): + if kwargs.get("app_id"): + if not AppManager.query_app(app_id=kwargs.get("app_id")): + raise NoFoundAppid(app_id=kwargs.get("app_id")) + return func(*args, **kwargs) + return _wrapper + + @staticmethod + def check_app_type(func): + @wraps(func) + def _wrapper(*args, **kwargs): + if kwargs.get("app_id"): + app_info = AppManager.query_app(app_id=kwargs.get("app_id")) + if not app_info: + raise NoFoundAppid(app_id=kwargs.get("app_id")) + role = kwargs.get("role") + if role == "super_client": + role = "client" + if role != app_info[0].f_app_type: + raise RoleTypeError(role=kwargs.get("role")) + return func(*args, **kwargs) + return _wrapper \ No newline at end of file diff --git a/python/fate_flow/db/config_manager.py b/python/fate_flow/manager/service/config_manager.py similarity index 63% rename from python/fate_flow/db/config_manager.py rename to python/fate_flow/manager/service/config_manager.py index 5ff18eee6..c5db9ae41 100644 --- a/python/fate_flow/db/config_manager.py +++ b/python/fate_flow/manager/service/config_manager.py @@ -13,19 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from .runtime_config import RuntimeConfig -from .service_registry import ServerRegistry -from .job_default_config import JobDefaultConfig -from fate_flow.manager.resource_manager import ResourceManager +from fate_flow.manager.service.resource_manager import ResourceManager +from fate_flow.manager.service.service_manager import ServerRegistry +from fate_flow.runtime.job_default_config import JobDefaultConfig class ConfigManager: @classmethod def load(cls): - configs = { - "job_default_config": JobDefaultConfig.load(), - "server_registry": ServerRegistry.load(), - } + JobDefaultConfig.load() ResourceManager.initialize() - RuntimeConfig.load_config_manager() - return configs \ No newline at end of file + ServerRegistry.load() diff --git a/python/fate_flow/manager/service/provider_manager.py b/python/fate_flow/manager/service/provider_manager.py new file mode 100644 index 000000000..388f1e250 --- /dev/null +++ b/python/fate_flow/manager/service/provider_manager.py @@ -0,0 +1,210 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import sys +from typing import Union + +from fate_flow.db import ProviderInfo, ComponentInfo +from fate_flow.db.base_models import DB, BaseModelOperate +from fate_flow.entity.spec.flow import ProviderSpec, LocalProviderSpec, DockerProviderSpec, K8sProviderSpec +from fate_flow.entity.types import ProviderDevice, PROTOCOL +from fate_flow.hub.flow_hub import FlowHub +from fate_flow.hub.provider import EntrypointABC +from fate_flow.runtime.system_settings import DEFAULT_FATE_PROVIDER_PATH, DEFAULT_PROVIDER, FATE_FLOW_PROVIDER_PATH +from fate_flow.runtime.component_provider import ComponentProvider +from fate_flow.utils.log import getLogger +from fate_flow.utils.version import get_versions, get_default_fate_version, get_flow_version +from fate_flow.utils.wraps_utils import filter_parameters + +stat_logger = getLogger("fate_flow_stat") + + +class ProviderManager(BaseModelOperate): + @classmethod + def get_provider_by_provider_name(cls, provider_name) -> ComponentProvider: + name, version, device = cls.parser_provider_name(provider_name) + provider_list = [provider_info for provider_info in cls.query_provider(name=name, version=version, device=device)] + if not provider_list: + raise ValueError(f"Query provider info failed: {provider_name}") + provider_info = provider_list[0] + return cls.get_provider(provider_info.f_name, provider_info.f_device, provider_info.f_version, + provider_info.f_metadata) + + @classmethod + def get_provider(cls, name, device, version, metadata, check=False) -> Union[ComponentProvider, None]: + if device == ProviderDevice.LOCAL: + metadata = LocalProviderSpec(check, **metadata) + elif device == ProviderDevice.DOCKER: + metadata = DockerProviderSpec(check, **metadata) + elif device == ProviderDevice.K8S: + metadata = K8sProviderSpec(check, **metadata) + else: + return None + return ComponentProvider(ProviderSpec(name=name, device=device, version=version, metadata=metadata)) + + @classmethod + @DB.connection_context() + def register_provider(cls, provider: ComponentProvider, components_description=None, protocol=PROTOCOL.FATE_FLOW): + if not components_description: + components_description = {} + provider_info = ProviderInfo() + provider_info.f_provider_name = provider.provider_name + provider_info.f_name = provider.name + provider_info.f_device = provider.device + provider_info.f_version = provider.version + provider_info.f_metadata = provider.metadata.dict() + operator_type = cls.safe_save(ProviderInfo, defaults=provider_info.to_dict(), + f_provider_name=provider.provider_name) + cls.register_component(provider, components_description, protocol) + return operator_type + + @classmethod + def register_component(cls, provider: ComponentProvider, components_description, protocol): + if not protocol: + protocol = PROTOCOL.FATE_FLOW + + if not components_description: + components_description = {} + + component_list = [] + + if components_description: + component_list = components_description.keys() + else: + entrypoint = cls.load_entrypoint(provider) + if entrypoint: + component_list = entrypoint.component_list + for component_name in component_list: + component = ComponentInfo() + component.f_provider_name = provider.provider_name + component.f_name = provider.name + component.f_device = provider.device + component.f_version = provider.version + component.f_component_name = component_name + component.f_protocol = protocol + component.f_component_description = components_description.get(component_name) + cls.safe_save( + ComponentInfo, defaults=component.to_dict(), + **dict( + f_provider_name=provider.provider_name, + f_name=provider.name, + f_device=provider.device, + f_version=provider.version, + f_component_name=component_name, + f_protocol=protocol + ) + ) + + @classmethod + @filter_parameters() + def query_provider(cls, **kwargs): + return cls._query(ProviderInfo, **kwargs) + + @classmethod + @filter_parameters() + def delete_provider(cls, **kwargs): + result = cls._delete(ProviderInfo, **kwargs) + cls.delete_provider_component_info(**kwargs) + return result + + @classmethod + @filter_parameters() + def delete_provider_component_info(cls, **kwargs): + result = cls._delete(ComponentInfo, **kwargs) + return result + + @classmethod + def register_default_providers(cls): + # register fate flow + cls.register_provider(cls.get_fate_flow_provider()) + # try to register fate + try: + cls.register_provider(cls.get_default_fate_provider()) + except Exception as e: + stat_logger.exception(e) + + @classmethod + def get_all_components(cls): + component_list = cls._query(ComponentInfo, force=True) + return list(set([component.f_component_name for component in component_list])) + + @classmethod + def get_flow_components(cls): + component_list = cls._query(ComponentInfo, name="fate_flow", force=True) + return list(set([component.f_component_name for component in component_list])) + + @classmethod + @filter_parameters() + def query_component_description(cls, **kwargs): + descriptions = {} + for info in cls._query(ComponentInfo, **kwargs): + descriptions[info.f_component_name] = info.f_component_description + return descriptions + + @classmethod + def get_fate_flow_provider(cls): + return cls.get_provider( + name="fate_flow", + version=get_flow_version(), + device=ProviderDevice.LOCAL, + metadata={ + "path": FATE_FLOW_PROVIDER_PATH, + "venv": sys.executable + }) + + @classmethod + def get_default_fate_provider(cls): + return cls.get_provider( + name="fate", + version=get_default_fate_version(), + device=ProviderDevice.LOCAL, + metadata={ + "path": DEFAULT_FATE_PROVIDER_PATH, + "venv": sys.executable + }) + + @classmethod + def generate_provider_name(cls, name, version, device): + return f"{name}:{version}@{device}" + + @classmethod + def parser_provider_name(cls, provider_name): + if not provider_name: + return None, None, None + try: + return provider_name.split(":")[0], provider_name.split(":")[1].split("@")[0], provider_name.split(":")[1].split("@")[1] + except: + raise ValueError(f"Provider format should be name:version@device, please check: {provider_name}") + + @classmethod + def check_provider_name(cls, provider_name): + name, version, device = cls.parser_provider_name(provider_name) + if not name or name == "*": + name = DEFAULT_PROVIDER.get("name") + if not version or version == "*": + if not DEFAULT_PROVIDER.get("version"): + DEFAULT_PROVIDER["version"] = get_versions().get(name.upper()) + version = DEFAULT_PROVIDER.get("version") + if not device or device == "*": + device = DEFAULT_PROVIDER.get("device") + provider_info = cls.query_provider(name=name, version=version, device=device) + if not provider_info: + raise ValueError(f"Not found provider[{cls.generate_provider_name(name, version, device)}]") + return cls.generate_provider_name(name, version, device) + + @staticmethod + def load_entrypoint(provider) -> Union[None, EntrypointABC]: + return FlowHub.load_provider_entrypoint(provider) diff --git a/python/fate_flow/manager/service/resource_manager.py b/python/fate_flow/manager/service/resource_manager.py new file mode 100644 index 000000000..55aebdb97 --- /dev/null +++ b/python/fate_flow/manager/service/resource_manager.py @@ -0,0 +1,267 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from pydantic import typing + +from fate_flow.db.base_models import DB +from fate_flow.db.db_models import EngineRegistry, Job, Task +from fate_flow.entity.types import EngineType, ResourceOperation, LauncherType +from fate_flow.runtime.job_default_config import JobDefaultConfig +from fate_flow.runtime.system_settings import IGNORE_RESOURCE_ROLES, ENGINES +from fate_flow.utils import engine_utils, base_utils, job_utils +from fate_flow.utils.log import getLogger +from fate_flow.utils.log_utils import schedule_logger + +stat_logger = getLogger() + + +class ResourceManager(object): + @classmethod + def initialize(cls): + engines_config = engine_utils.get_engines_config_from_conf(group_map=True) + for engine_type, engine_configs in engines_config.items(): + for engine_name, engine_config in engine_configs.items(): + cls.register_engine(engine_type=engine_type, engine_name=engine_name, engine_config=engine_config) + + @classmethod + @DB.connection_context() + def register_engine(cls, engine_type, engine_name, engine_config): + cores = engine_config.get("cores", 0) + memory = engine_config.get("memory", 0) + filters = [EngineRegistry.f_engine_type == engine_type, EngineRegistry.f_engine_name == engine_name] + resources = EngineRegistry.select().where(*filters) + if resources: + resource = resources[0] + update_fields = {} + update_fields[EngineRegistry.f_engine_config] = engine_config + update_fields[EngineRegistry.f_cores] = cores + update_fields[EngineRegistry.f_memory] = memory + update_fields[EngineRegistry.f_remaining_cores] = EngineRegistry.f_remaining_cores + ( + cores - resource.f_cores) + update_fields[EngineRegistry.f_remaining_memory] = EngineRegistry.f_remaining_memory + ( + memory - resource.f_memory) + operate = EngineRegistry.update(update_fields).where(*filters) + update_status = operate.execute() > 0 + if update_status: + stat_logger.info(f"update {engine_type} engine {engine_name} registration information") + else: + stat_logger.info(f"update {engine_type} engine {engine_name} registration information takes no effect") + else: + resource = EngineRegistry() + resource.f_create_time = base_utils.current_timestamp() + resource.f_engine_type = engine_type + resource.f_engine_name = engine_name + resource.f_engine_config = engine_config + + resource.f_cores = cores + resource.f_memory = memory + resource.f_remaining_cores = cores + resource.f_remaining_memory = memory + try: + resource.save(force_insert=True) + except Exception as e: + stat_logger.warning(e) + stat_logger.info(f"create {engine_type} engine {engine_name} registration information") + + @classmethod + def apply_for_job_resource(cls, job_id, role, party_id): + return cls.resource_for_job(job_id=job_id, role=role, party_id=party_id, operation_type=ResourceOperation.APPLY) + + @classmethod + def return_job_resource(cls, job_id, role, party_id): + return cls.resource_for_job(job_id=job_id, role=role, party_id=party_id, + operation_type=ResourceOperation.RETURN) + + @classmethod + @DB.connection_context() + def resource_for_job(cls, job_id, role, party_id, operation_type: ResourceOperation): + operate_status = False + cores, memory = cls.query_job_resource(job_id=job_id, role=role, party_id=party_id) + engine_name = ENGINES.get(EngineType.COMPUTING) + try: + with DB.atomic(): + updates = { + Job.f_engine_name: engine_name, + Job.f_cores: cores, + Job.f_memory: memory, + } + filters = [ + Job.f_job_id == job_id, + Job.f_role == role, + Job.f_party_id == party_id, + ] + if operation_type is ResourceOperation.APPLY: + updates[Job.f_resource_in_use] = True + updates[Job.f_apply_resource_time] = base_utils.current_timestamp() + filters.append(Job.f_resource_in_use == False) + elif operation_type is ResourceOperation.RETURN: + updates[Job.f_resource_in_use] = False + updates[Job.f_return_resource_time] = base_utils.current_timestamp() + filters.append(Job.f_resource_in_use == True) + operate = Job.update(updates).where(*filters) + record_status = operate.execute() > 0 + if not record_status: + raise RuntimeError(f"record job {job_id} resource {operation_type} failed on {role} {party_id}") + + if cores or memory: + filters, updates = cls.update_resource_sql(resource_model=EngineRegistry, + cores=cores, + memory=memory, + operation_type=operation_type, + ) + filters.append(EngineRegistry.f_engine_name == engine_name) + operate = EngineRegistry.update(updates).where(*filters) + apply_status = operate.execute() > 0 + else: + apply_status = True + if not apply_status: + raise RuntimeError( + f"update engine {engine_name} record for job {job_id} resource {operation_type} on {role} {party_id} failed") + operate_status = True + except Exception as e: + schedule_logger(job_id).warning(e) + schedule_logger(job_id).warning( + f"{operation_type} job resource(cores {cores} memory {memory}) on {role} {party_id} failed") + operate_status = False + finally: + remaining_cores, remaining_memory = cls.get_remaining_resource(EngineRegistry, + [ + EngineRegistry.f_engine_type == EngineType.COMPUTING, + EngineRegistry.f_engine_name == engine_name]) + operate_msg = "successfully" if operate_status else "failed" + schedule_logger(job_id).info( + f"{operation_type} job resource(cores {cores} memory {memory}) on {role} {party_id} {operate_msg}, remaining cores: {remaining_cores} remaining memory: {remaining_memory}") + return operate_status + + @classmethod + def apply_for_task_resource(cls, **task_info): + return ResourceManager.resource_for_task(task_info=task_info, operation_type=ResourceOperation.APPLY) + + @classmethod + def return_task_resource(cls, **task_info): + return ResourceManager.resource_for_task(task_info=task_info, operation_type=ResourceOperation.RETURN) + + @classmethod + @DB.connection_context() + def resource_for_task(cls, task_info, operation_type): + cores_per_task, memory_per_task = cls.query_task_resource(task_info=task_info) + schedule_logger(task_info["job_id"]).info(f"{operation_type} cores_per_task:{cores_per_task}, memory_per_task:{memory_per_task}") + operate_status = False + if cores_per_task or memory_per_task: + try: + with DB.atomic(): + updates = {} + filters = [ + Task.f_job_id == task_info["job_id"], + Task.f_role == task_info["role"], + Task.f_party_id == task_info["party_id"], + Task.f_task_id == task_info["task_id"], + Task.f_task_version == task_info["task_version"] + ] + if operation_type is ResourceOperation.APPLY: + updates[Task.f_resource_in_use] = True + filters.append(Task.f_resource_in_use == False) + elif operation_type is ResourceOperation.RETURN: + updates[Task.f_resource_in_use] = False + filters.append(Task.f_resource_in_use == True) + operate = Task.update(updates).where(*filters) + record_status = operate.execute() > 0 + if not record_status: + raise RuntimeError(f"record task {task_info['task_id']} {task_info['task_version']} resource" + f"{operation_type} failed on {task_info['role']} {task_info['party_id']}") + filters, updates = cls.update_resource_sql(resource_model=Job, + cores=cores_per_task, + memory=memory_per_task, + operation_type=operation_type, + ) + filters.append(Job.f_job_id == task_info["job_id"]) + filters.append(Job.f_role == task_info["role"]) + filters.append(Job.f_party_id == task_info["party_id"]) + # filters.append(Job.f_resource_in_use == True) + operate = Job.update(updates).where(*filters) + operate_status = operate.execute() > 0 + if not operate_status: + raise RuntimeError(f"record task {task_info['task_id']} {task_info['task_version']} job resource " + f"{operation_type} failed on {task_info['role']} {task_info['party_id']}") + except Exception as e: + schedule_logger(task_info["job_id"]).warning(e) + operate_status = False + else: + operate_status = True + if operate_status: + schedule_logger(task_info["job_id"]).info( + "task {} {} {} {} {} resource successfully".format( + task_info["role"], + task_info["party_id"], + task_info["task_id"], + task_info["task_version"], + operation_type)) + else: + schedule_logger(task_info["job_id"]).warning( + "task {} {} {} resource failed".format( + task_info["task_id"], + task_info["task_version"], operation_type)) + return operate_status + + @classmethod + def query_job_resource(cls, job_id, role, party_id): + cores, memory = job_utils.get_job_resource_info(job_id, role, party_id) + return int(cores), memory + + @classmethod + def query_task_resource(cls, task_info: dict = None): + cores_per_task = 0 + memory_per_task = 0 + if task_info["role"] in IGNORE_RESOURCE_ROLES: + return cores_per_task, memory_per_task + task_cores, memory, launcher_name = job_utils.get_task_resource_info( + task_info["job_id"], task_info["role"], task_info["party_id"], task_info["task_id"], task_info["task_version"] + ) + if launcher_name == LauncherType.DEEPSPEED: + # todo: apply gpus + return 0, 0 + return task_cores, memory + + @classmethod + def update_resource_sql(cls, resource_model: typing.Union[EngineRegistry, Job], cores, memory, operation_type: ResourceOperation): + if operation_type is ResourceOperation.APPLY: + filters = [ + resource_model.f_remaining_cores >= cores, + resource_model.f_remaining_memory >= memory + ] + updates = {resource_model.f_remaining_cores: resource_model.f_remaining_cores - cores, + resource_model.f_remaining_memory: resource_model.f_remaining_memory - memory} + elif operation_type is ResourceOperation.RETURN: + filters = [ + resource_model.f_remaining_cores + cores <= resource_model.f_cores + ] + updates = {resource_model.f_remaining_cores: resource_model.f_remaining_cores + cores, + resource_model.f_remaining_memory: resource_model.f_remaining_memory + memory} + else: + raise RuntimeError(f"can not support {operation_type} resource operation type") + return filters, updates + + @classmethod + def get_remaining_resource(cls, resource_model: typing.Union[EngineRegistry, Job], filters): + remaining_cores, remaining_memory = None, None + try: + objs = resource_model.select(resource_model.f_remaining_cores, resource_model.f_remaining_memory).where( + *filters) + if objs: + remaining_cores, remaining_memory = objs[0].f_remaining_cores, objs[0].f_remaining_memory + except Exception as e: + schedule_logger().exception(e) + finally: + return remaining_cores, remaining_memory diff --git a/python/fate_flow/db/db_services.py b/python/fate_flow/manager/service/service_manager.py similarity index 61% rename from python/fate_flow/db/db_services.py rename to python/fate_flow/manager/service/service_manager.py index 74800365c..65bcf7080 100644 --- a/python/fate_flow/db/db_services.py +++ b/python/fate_flow/manager/service/service_manager.py @@ -16,8 +16,10 @@ import abc import atexit import json +import socket import time from functools import wraps +from pathlib import Path from queue import Queue from threading import Thread from urllib import parse @@ -27,17 +29,19 @@ from kazoo.security import make_digest_acl from shortuuid import ShortUUID -from fate_arch.common.versions import get_fate_version - -from fate_flow.db.service_registry import ServerRegistry -from fate_flow.entity.instance import FlowInstance -from fate_flow.errors.error_services import * -from fate_flow.settings import ( - FATE_FLOW_MODEL_TRANSFER_ENDPOINT, GRPC_PORT, HOST, HTTP_PORT, NGINX_HOST, NGINX_HTTP_PORT, - RANDOM_INSTANCE_ID, USE_REGISTRY, ZOOKEEPER, ZOOKEEPER_REGISTRY, stat_logger, -) -from fate_flow.utils.model_utils import models_group_by_party_model_id_and_model_version +from fate_flow.db import ServiceRegistryInfo, ServerRegistryInfo +from fate_flow.db.base_models import DB +from fate_flow.entity.types import FlowInstance +from fate_flow.errors.zookeeper_error import ServiceNotSupported, ServicesError, ZooKeeperNotConfigured, \ + MissingZooKeeperUsernameOrPassword, ZooKeeperBackendError +from fate_flow.runtime.reload_config_base import ReloadConfigBase +from fate_flow.runtime.system_settings import RANDOM_INSTANCE_ID, HOST, HTTP_PORT, GRPC_PORT, ZOOKEEPER_REGISTRY, \ + ZOOKEEPER, USE_REGISTRY, NGINX_HOST, NGINX_HTTP_PORT, FATE_FLOW_MODEL_TRANSFER_ENDPOINT, SERVICE_CONF_NAME +from fate_flow.utils import conf_utils, file_utils +from fate_flow.utils.log import getLogger +from fate_flow.utils.version import get_flow_version +stat_logger = getLogger("fate_flow_stat") model_download_endpoint = f'http://{NGINX_HOST}:{NGINX_HTTP_PORT}{FATE_FLOW_MODEL_TRANSFER_ENDPOINT}' @@ -47,7 +51,7 @@ json.dumps({ 'instance_id': instance_id, 'timestamp': round(time.time() * 1000), - 'version': get_fate_version() or '', + 'version': get_flow_version() or '', 'host': HOST, 'grpc_port': GRPC_PORT, 'http_port': HTTP_PORT, @@ -72,21 +76,6 @@ def magic(self, service_name, *args, **kwargs): return magic -def get_model_download_url(party_model_id, model_version): - """Get the full url of model download. - - :param str party_model_id: The party model id, `#` will be replaced with `~`. - :param str model_version: The model version. - :return: The download url. - :rtype: str - """ - return '{endpoint}/{model_id}/{model_version}'.format( - endpoint=model_download_endpoint, - model_id=party_model_id.replace('#', '~'), - model_version=model_version, - ) - - class ServicesDB(abc.ABC): """Database for storage service urls. Abstract base class for the real backends. @@ -138,14 +127,8 @@ def delete(self, service_name, service_url): stat_logger.exception(e) def register_model(self, party_model_id, model_version): - """Call `self.insert` for insert a service url to database. - Currently, only `fateflow` (model download) urls are supported. - - :param str party_model_id: The party model id, `#` will be replaced with `_`. - :param str model_version: The model version. - :return: None - """ - self.insert('fateflow', get_model_download_url(party_model_id, model_version)) + # todo + pass def unregister_model(self, party_model_id, model_version): """Call `self.delete` for delete a service url from database. @@ -155,7 +138,8 @@ def unregister_model(self, party_model_id, model_version): :param str model_version: The model version. :return: None """ - self.delete('fateflow', get_model_download_url(party_model_id, model_version)) + # todo + pass def register_flow(self): """Call `self.insert` for insert the flow server address to databae. @@ -197,22 +181,25 @@ def register_models(self): :return: None """ - for model in models_group_by_party_model_id_and_model_version(): - self.register_model(model.f_party_model_id, model.f_model_version) + # todo: + pass def unregister_models(self): """Unregister all service urls of each model to database on this node. :return: None """ - for model in models_group_by_party_model_id_and_model_version(): - self.unregister_model(model.f_party_model_id, model.f_model_version) + # todo + pass - def get_servers(self): + def get_servers(self, to_dict=False): servers = {} for znode, value in self.get_urls('flow-server', True): instance = FlowInstance(**json.loads(value)) - servers[instance.instance_id] = instance + _id = instance.instance_id + if to_dict: + instance = instance.to_dict() + servers[_id] = instance return servers @@ -283,10 +270,6 @@ def _get_znode_path(self, service_name, service_url): :return: The znode path composed of `self.znodes[service_name]` and escaped `service_url`. :rtype: str - :example: - - >>> self._get_znode_path('fateflow', 'http://127.0.0.1:9380/v1/model/transfer/arbiter-10000_guest-9999_host-10000_model/202105060929263278441') - '/FATE-SERVICES/flow/online/transfer/providers/http%3A%2F%2F127.0.0.1%3A9380%2Fv1%2Fmodel%2Ftransfer%2Farbiter-10000_guest-9999_host-10000_model%2F202105060929263278441' """ return '/'.join([self.znodes[service_name], parse.quote(service_url, safe='')]) @@ -373,6 +356,151 @@ def _get_urls(self, service_name, with_values=False): return [(url, '') for url in urls] if with_values else urls +class ServerRegistry(ReloadConfigBase): + @classmethod + def load(cls): + cls.load_server_info_from_conf() + cls.load_server_info_from_db() + + @classmethod + def register(cls, server_name, host, port, protocol): + server_name = server_name.upper() + server_info = { + "host": host, + "port": port, + "protocol": protocol + } + cls.save_server_info_to_db(server_name, host, port=port, protocol=protocol) + setattr(cls, server_name, server_info) + server_info.update({"server_name": server_name}) + return server_info + + @classmethod + def delete_server_from_db(cls, server_name): + operate = ServerRegistryInfo.delete().where(ServerRegistryInfo.f_server_name == server_name.upper()) + return operate.execute() + + @classmethod + def parameter_check(cls, service_info): + if "host" in service_info and "port" in service_info: + cls.connection_test(service_info.get("host"), service_info.get("port")) + + @classmethod + def connection_test(cls, ip, port): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = s.connect_ex((ip, port)) + if result != 0: + raise ConnectionRefusedError(f"connection refused: host {ip}, port {port}") + + @classmethod + def query(cls, service_name, default=None): + service_info = getattr(cls, service_name, default) + if not service_info: + service_info = conf_utils.get_base_config(service_name, default) + return service_info + + @classmethod + @DB.connection_context() + def query_server_info_from_db(cls, server_name=None) -> [ServerRegistryInfo]: + if server_name: + server_list = ServerRegistryInfo.select().where(ServerRegistryInfo.f_server_name == server_name.upper()) + else: + server_list = ServerRegistryInfo.select() + return [server for server in server_list] + + @classmethod + @DB.connection_context() + def load_server_info_from_db(cls): + for server in cls.query_server_info_from_db(): + server_info = { + "host": server.f_host, + "port": server.f_port, + "protocol": server.f_protocol + } + setattr(cls, server.f_server_name.upper(), server_info) + + @classmethod + def load_server_info_from_conf(cls): + path = Path(file_utils.get_fate_flow_directory()) / 'conf' / SERVICE_CONF_NAME + conf = file_utils.load_yaml_conf(path) + if not isinstance(conf, dict): + raise ValueError('invalid config file') + + local_path = path.with_name(f'local.{SERVICE_CONF_NAME}') + if local_path.exists(): + local_conf = file_utils.load_yaml_conf(local_path) + if not isinstance(local_conf, dict): + raise ValueError('invalid local config file') + conf.update(local_conf) + for k, v in conf.items(): + if isinstance(v, dict): + setattr(cls, k.upper(), v) + + @classmethod + @DB.connection_context() + def save_server_info_to_db(cls, server_name, host, port, protocol="http"): + server_info = { + "f_server_name": server_name, + "f_host": host, + "f_port": port, + "f_protocol": protocol + } + entity_model, status = ServerRegistryInfo.get_or_create( + f_server_name=server_name, + defaults=server_info) + if status is False: + for key in server_info: + setattr(entity_model, key, server_info[key]) + entity_model.save(force_insert=False) + + +class ServiceRegistry: + @classmethod + @DB.connection_context() + def load_service(cls, server_name, service_name) -> [ServiceRegistryInfo]: + server_name = server_name.upper() + service_registry_list = ServiceRegistryInfo.query(server_name=server_name, service_name=service_name) + return [service for service in service_registry_list] + + @classmethod + @DB.connection_context() + def save_service_info(cls, server_name, service_name, uri, method="POST", server_info=None, params=None, data=None, headers=None, protocol="http"): + server_name = server_name.upper() + if not server_info: + server_list = ServerRegistry.query_server_info_from_db(server_name=server_name) + if not server_list: + raise Exception(f"no found server {server_name}") + server_info = server_list[0] + url = f"{server_info.f_protocol}://{server_info.f_host}:{server_info.f_port}{uri}" + else: + url = f"{server_info.get('protocol', protocol)}://{server_info.get('host')}:{server_info.get('port')}{uri}" + service_info = { + "f_server_name": server_name, + "f_service_name": service_name, + "f_url": url, + "f_method": method, + "f_params": params if params else {}, + "f_data": data if data else {}, + "f_headers": headers if headers else {} + } + entity_model, status = ServiceRegistryInfo.get_or_create( + f_server_name=server_name, + f_service_name=service_name, + defaults=service_info) + if status is False: + for key in service_info: + setattr(entity_model, key, service_info[key]) + entity_model.save(force_insert=False) + + @classmethod + @DB.connection_context() + def delete(cls, server_name, service_name): + server_name = server_name.upper() + operate = ServiceRegistryInfo.delete().where(ServiceRegistryInfo.f_server_name == server_name.upper(), + ServiceRegistryInfo.f_service_name == service_name) + return operate.execute() + + def service_db(): """Initialize services database. Currently only ZooKeeper is supported. diff --git a/python/fate_flow/manager/service/worker_manager.py b/python/fate_flow/manager/service/worker_manager.py new file mode 100644 index 000000000..eed2f4152 --- /dev/null +++ b/python/fate_flow/manager/service/worker_manager.py @@ -0,0 +1,168 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import io +import logging +import os +import subprocess +import sys +from uuid import uuid1 + +from fate_flow.runtime.system_settings import ENGINES +from ruamel import yaml + +from fate_flow.db.base_models import DB, auto_date_timestamp_db_field +from fate_flow.db.db_models import Task, WorkerInfo +from fate_flow.entity.types import WorkerName, EngineType, ComputingEngine +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.utils import job_utils, process_utils +from fate_flow.utils.base_utils import current_timestamp, json_dumps +from fate_flow.utils.log_utils import failed_log, schedule_logger, start_log, successful_log + + +class WorkerManager: + @classmethod + def start_task_worker(cls, worker_name, task_info, task_parameters=None, executable=None, common_cmd=None, + extra_env: dict = None, record=False, stderr=None, sync=False, config_dir=None, std_dir=None, + **kwargs): + if not extra_env: + extra_env = {} + worker_id = uuid1().hex + if not config_dir or not std_dir: + config_dir, std_dir = cls.get_process_dirs( + job_id=task_info.get("job_id"), + role=task_info.get("role"), + party_id=task_info.get("party_id"), + task_name=task_info.get("task_name"), + task_version=task_info.get("task_version") + ) + params_env = {} + if task_parameters: + params_env = cls.get_env(task_info.get("job_id"), task_parameters) + extra_env.update(params_env) + if executable: + process_cmd = executable + else: + process_cmd = [os.getenv("EXECUTOR_ENV") or sys.executable or "python3"] + process_cmd.extend(common_cmd) + if sync and cls.worker_outerr_with_pipe(worker_name): + stderr = subprocess.PIPE + p = process_utils.run_subprocess(job_id=task_info.get("job_id"), config_dir=config_dir, process_cmd=process_cmd, + added_env=extra_env, std_dir=std_dir, cwd_dir=config_dir, + process_name=worker_name.value, stderr=stderr) + if record: + cls.save_worker_info(task_info=task_info, worker_name=worker_name, worker_id=worker_id, + run_ip=RuntimeConfig.JOB_SERVER_HOST, run_pid=p.pid, config=task_parameters, + cmd=process_cmd) + return { + "run_pid": p.pid, + "run_ip": RuntimeConfig.JOB_SERVER_HOST, + "worker_id": worker_id, + "cmd": process_cmd, + "run_port": RuntimeConfig.HTTP_PORT + } + else: + if sync: + error_io = io.BytesIO() + if cls.worker_outerr_with_pipe(worker_name): + while True: + output = p.stderr.readline() + if output == b'' and p.poll() is not None: + break + if output: + error_io.write(output) + error_io.seek(0) + _code = p.wait() + _e = error_io.read() + if _e and _code: + logging.error(f"process {worker_name.value} run error[code:{_code}]\n: {_e.decode()}") + return p + + @classmethod + def worker_outerr_with_pipe(cls, worker_name): + return worker_name == WorkerName.TASK_EXECUTE and \ + ENGINES.get(EngineType.COMPUTING) not in [ComputingEngine.SPARK] + + @classmethod + def get_process_dirs(cls, job_id, role, party_id, task_name, task_version): + config_dir = job_utils.get_job_directory(job_id, role, party_id, task_name, str(task_version)) + std_dir = job_utils.get_job_log_directory(job_id, role, party_id, task_name, "stdout") + os.makedirs(config_dir, exist_ok=True) + return config_dir, std_dir + + @classmethod + def get_config(cls, config_dir, config): + config_path = os.path.join(config_dir, "config.json") + with open(config_path, 'w') as fw: + fw.write(json_dumps(config, indent=True)) + result_path = os.path.join(config_dir, "result.json") + return config_path, result_path + + @classmethod + def get_env(cls, job_id, task_parameters): + env = { + "FATE_JOB_ID": job_id, + "FATE_TASK_CONFIG": yaml.dump(task_parameters), + } + return env + + @classmethod + def cmd_to_func_kwargs(cls, cmd): + kwargs = {} + for i in range(2, len(cmd), 2): + kwargs[cmd[i].lstrip("--")] = cmd[i+1] + return kwargs + + @classmethod + @DB.connection_context() + def save_worker_info(cls, task_info, worker_name: WorkerName, worker_id, **kwargs): + worker = WorkerInfo() + ignore_attr = auto_date_timestamp_db_field() + for attr, value in task_info.items(): + attr = f"f_{attr}" + if hasattr(worker, attr) and attr not in ignore_attr and value is not None: + setattr(worker, attr, value) + worker.f_create_time = current_timestamp() + worker.f_worker_name = worker_name.value + worker.f_worker_id = worker_id + for k, v in kwargs.items(): + attr = f"f_{k}" + if hasattr(worker, attr) and v is not None: + setattr(worker, attr, v) + rows = worker.save(force_insert=True) + if rows != 1: + raise Exception("save worker info failed") + + @classmethod + @DB.connection_context() + def kill_task_all_workers(cls, task: Task): + schedule_logger(task.f_job_id).info(start_log("kill all workers", task=task)) + workers_info = WorkerInfo.query(task_id=task.f_task_id, task_version=task.f_task_version, role=task.f_role, + party_id=task.f_party_id) + for worker_info in workers_info: + schedule_logger(task.f_job_id).info( + start_log(f"kill {worker_info.f_worker_name}({worker_info.f_run_pid})", task=task)) + try: + cls.kill_worker(worker_info) + schedule_logger(task.f_job_id).info( + successful_log(f"kill {worker_info.f_worker_name}({worker_info.f_run_pid})", task=task)) + except Exception as e: + schedule_logger(task.f_job_id).warning( + failed_log(f"kill {worker_info.f_worker_name}({worker_info.f_run_pid})", task=task), exc_info=True) + schedule_logger(task.f_job_id).info(successful_log("kill all workers", task=task)) + + @classmethod + def kill_worker(cls, worker_info: WorkerInfo): + process_utils.kill_process(pid=worker_info.f_run_pid, expected_cmdline=worker_info.f_cmd) diff --git a/python/fate_flow/manager/worker/__init__.py b/python/fate_flow/manager/worker/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/manager/worker/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/manager/worker/deepspeed_download_model.py b/python/fate_flow/manager/worker/deepspeed_download_model.py new file mode 100644 index 000000000..1f449b068 --- /dev/null +++ b/python/fate_flow/manager/worker/deepspeed_download_model.py @@ -0,0 +1,50 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import argparse + +from fate_flow.engine.devices import build_engine +from fate_flow.entity.types import LauncherType +from fate_flow.manager.operation.job_saver import JobSaver +from fate_flow.utils.log_utils import schedule_logger + + +class DownloadModel(object): + def run(self, args): + tasks = JobSaver.query_task( + task_id=args.task_id, + task_version=args.task_version, + job_id=args.job_id, + role=args.role, + party_id=args.party_id + ) + task = tasks[0] + deepspeed_engine = build_engine(task.f_provider_name, LauncherType.DEEPSPEED) + schedule_logger(args.job_id).info("start download model") + deepspeed_engine.download_model_do(task=task, path=args.path, worker_id=task.f_worker_id) + schedule_logger(args.job_id).info("download model success") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--path', required=True, type=str, help="path") + parser.add_argument('--job_id', required=True, type=str, help="job id") + parser.add_argument('--role', required=True, type=str, help="role") + parser.add_argument('--party_id', required=True, type=str, help="party id") + parser.add_argument('--task_id', required=True, type=str, help="task id") + parser.add_argument('--task_version', required=True, type=int, help="task version") + + args = parser.parse_args() + DownloadModel().run(args) diff --git a/python/fate_flow/manager/worker/fate_ds_executor.py b/python/fate_flow/manager/worker/fate_ds_executor.py new file mode 100644 index 000000000..71e2d2de8 --- /dev/null +++ b/python/fate_flow/manager/worker/fate_ds_executor.py @@ -0,0 +1,49 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import os +import sys + + +class FateSubmit: + @staticmethod + def run(): + import runpy + runpy.run_module(mod_name='fate.components', run_name='__main__') + + +if __name__ == "__main__": + # replace deepspeed rank env + print(os.environ.get("EGGROLL_DEEPSPEED_CONTAINER_LOGS_DIR")) + print(os.environ.get("EGGROLL_DEEPSPEED_CONTAINER_MODELS_DIR")) + print(os.environ.get("EGGROLL_DEEPSPEED_CONTAINER_RESULT_DIR")) + + result_index = sys.argv.index("--execution-final-meta-path") + 1 + result_path = os.environ.get(sys.argv[result_index], "") + + env_name_index = sys.argv.index("--env-name") + 1 + env_key = sys.argv[env_name_index] + sys.argv[result_index] = sys.argv[result_index].replace( + os.environ.get("DEEPSPEED_RESULT_PLACEHOLDER"), + os.environ.get("EGGROLL_DEEPSPEED_CONTAINER_RESULT_DIR") + ) + + env_str = os.environ.get(sys.argv[env_name_index], "") + env_str = env_str.replace(os.environ.get("DEEPSPEED_LOGS_DIR_PLACEHOLDER"), os.environ.get("EGGROLL_DEEPSPEED_CONTAINER_LOGS_DIR")) + env_str = env_str.replace(os.environ.get("DEEPSPEED_MODEL_DIR_PLACEHOLDER"), os.environ.get("EGGROLL_DEEPSPEED_CONTAINER_MODELS_DIR")) + print(json.loads(env_str)) + os.environ[env_key] = env_str + FateSubmit.run() diff --git a/python/fate_flow/manager/worker/fate_executor.py b/python/fate_flow/manager/worker/fate_executor.py new file mode 100644 index 000000000..da1e79481 --- /dev/null +++ b/python/fate_flow/manager/worker/fate_executor.py @@ -0,0 +1,26 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +class FateSubmit: + @staticmethod + def run(): + import runpy + runpy.run_module(mod_name='fate.components', run_name='__main__') + + +if __name__ == "__main__": + FateSubmit.run() diff --git a/python/fate_flow/manager/worker/fate_flow_executor.py b/python/fate_flow/manager/worker/fate_flow_executor.py new file mode 100644 index 000000000..ac9c64a09 --- /dev/null +++ b/python/fate_flow/manager/worker/fate_flow_executor.py @@ -0,0 +1,31 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import argparse +import sys + +from fate_flow.entity import BaseEntity +from fate_flow.utils.log import getLogger + + +class FateFlowSubmit: + @staticmethod + def run(): + import runpy + runpy.run_module(mod_name='fate_flow.components', run_name='__main__') + + +if __name__ == "__main__": + FateFlowSubmit.run() diff --git a/python/fate_flow/manager/worker_manager.py b/python/fate_flow/manager/worker_manager.py deleted file mode 100644 index ab06b0fa9..000000000 --- a/python/fate_flow/manager/worker_manager.py +++ /dev/null @@ -1,291 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os -import subprocess -import sys -from uuid import uuid1 - -from fate_arch.common.base_utils import current_timestamp, json_dumps -from fate_arch.common.file_utils import load_json_conf -from fate_arch.metastore.base_model import auto_date_timestamp_db_field - -from fate_flow.db.db_models import DB, Task, WorkerInfo -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.entity import ComponentProvider, RunParameters -from fate_flow.entity.types import WorkerName -from fate_flow.settings import stat_logger -from fate_flow.utils import job_utils, process_utils -from fate_flow.utils.log_utils import failed_log, ready_log, schedule_logger, start_log, successful_log - - -class WorkerManager: - @classmethod - def start_general_worker(cls, worker_name: WorkerName, job_id="", role="", party_id=0, provider: ComponentProvider = None, - initialized_config: dict = None, run_in_subprocess=True, **kwargs): - if RuntimeConfig.DEBUG: - run_in_subprocess = True - participate = locals() - worker_id, config_dir, log_dir = cls.get_process_dirs(worker_name=worker_name, - job_id=job_id, - role=role, - party_id=party_id) - if worker_name in [WorkerName.PROVIDER_REGISTRAR, WorkerName.DEPENDENCE_UPLOAD]: - if not provider: - raise ValueError("no provider argument") - config = { - "provider": provider.to_dict() - } - if worker_name == WorkerName.PROVIDER_REGISTRAR: - from fate_flow.worker.provider_registrar import ProviderRegistrar - module = ProviderRegistrar - module_file_path = sys.modules[ProviderRegistrar.__module__].__file__ - specific_cmd = [] - elif worker_name == WorkerName.DEPENDENCE_UPLOAD: - from fate_flow.worker.dependence_upload import DependenceUpload - module = DependenceUpload - module_file_path = sys.modules[DependenceUpload.__module__].__file__ - specific_cmd = [ - '--dependence_type', kwargs.get("dependence_type") - ] - provider_info = provider.to_dict() - elif worker_name is WorkerName.TASK_INITIALIZER: - if not initialized_config: - raise ValueError("no initialized_config argument") - config = initialized_config - from fate_flow.worker.task_initializer import TaskInitializer - module = TaskInitializer - module_file_path = sys.modules[TaskInitializer.__module__].__file__ - specific_cmd = [] - provider_info = initialized_config["provider"] - else: - raise Exception(f"not support {worker_name} worker") - config_path, result_path = cls.get_config(config_dir=config_dir, config=config, log_dir=log_dir) - - process_cmd = [ - sys.executable or "python3", - module_file_path, - "--config", config_path, - '--result', result_path, - "--log_dir", log_dir, - "--parent_log_dir", os.path.dirname(log_dir), - "--worker_id", worker_id, - "--run_ip", RuntimeConfig.JOB_SERVER_HOST, - "--job_server", f"{RuntimeConfig.JOB_SERVER_HOST}:{RuntimeConfig.HTTP_PORT}", - ] - - if job_id: - process_cmd.extend([ - "--job_id", job_id, - "--role", role, - "--party_id", party_id, - ]) - - process_cmd.extend(specific_cmd) - if run_in_subprocess: - p = process_utils.run_subprocess(job_id=job_id, config_dir=config_dir, process_cmd=process_cmd, - added_env=cls.get_env(job_id, provider_info), log_dir=log_dir, - cwd_dir=config_dir, process_name=worker_name.value, process_id=worker_id) - participate["pid"] = p.pid - if job_id and role and party_id: - logger = schedule_logger(job_id) - msg = f"{worker_name} worker {worker_id} subprocess {p.pid}" - else: - logger = stat_logger - msg = f"{worker_name} worker {worker_id} subprocess {p.pid}" - logger.info(ready_log(msg=msg, role=role, party_id=party_id)) - - # asynchronous - if worker_name in [WorkerName.DEPENDENCE_UPLOAD]: - if kwargs.get("callback") and kwargs.get("callback_param"): - callback_param = {} - participate.update(participate.get("kwargs", {})) - for k, v in participate.items(): - if k in kwargs.get("callback_param"): - callback_param[k] = v - kwargs.get("callback")(**callback_param) - else: - try: - p.wait(timeout=120) - if p.returncode == 0: - logger.info(successful_log(msg=msg, role=role, party_id=party_id)) - else: - logger.info(failed_log(msg=msg, role=role, party_id=party_id)) - if p.returncode == 0: - return p.returncode, load_json_conf(result_path) - else: - std_path = process_utils.get_std_path(log_dir=log_dir, process_name=worker_name.value, process_id=worker_id) - raise Exception(f"run error, please check logs: {std_path}, {log_dir}/INFO.log") - except subprocess.TimeoutExpired as e: - err = failed_log(msg=f"{msg} run timeout", role=role, party_id=party_id) - logger.exception(err) - raise Exception(err) - finally: - try: - p.kill() - p.poll() - except Exception as e: - logger.exception(e) - else: - kwargs = cls.cmd_to_func_kwargs(process_cmd) - code, message, result = module().run(**kwargs) - if code == 0: - return code, result - else: - raise Exception(message) - - @classmethod - def start_task_worker(cls, worker_name, task: Task, task_parameters: RunParameters = None, - executable: list = None, extra_env: dict = None, **kwargs): - worker_id, config_dir, log_dir = cls.get_process_dirs(worker_name=worker_name, - job_id=task.f_job_id, - role=task.f_role, - party_id=task.f_party_id, - task=task) - if worker_name is WorkerName.TASK_EXECUTOR: - from fate_flow.worker.task_executor import TaskExecutor - module_file_path = sys.modules[TaskExecutor.__module__].__file__ - else: - raise Exception(f"not support {worker_name} worker") - - if task_parameters is None: - task_parameters = RunParameters(**job_utils.get_job_parameters(task.f_job_id, task.f_role, task.f_party_id)) - - config = task_parameters.to_dict() - config["src_user"] = kwargs.get("src_user") - env = cls.get_env(task.f_job_id, task.f_provider_info) - if executable: - process_cmd = executable - else: - process_cmd = [env.get("PYTHON_ENV") or sys.executable or "python3"] - common_cmd = [module_file_path] - common_cmd.extend(cls.generate_common_cmd(task, config_dir, config, log_dir, worker_id)[2]) - process_cmd.extend(common_cmd) - if extra_env: - env.update(extra_env) - schedule_logger(task.f_job_id).info( - f"task {task.f_task_id} {task.f_task_version} on {task.f_role} {task.f_party_id} {worker_name} worker subprocess is ready") - p = process_utils.run_subprocess(job_id=task.f_job_id, config_dir=config_dir, process_cmd=process_cmd, - added_env=env, log_dir=log_dir, cwd_dir=config_dir, process_name=worker_name.value, - process_id=worker_id) - cls.save_worker_info(task=task, worker_name=worker_name, worker_id=worker_id, run_ip=RuntimeConfig.JOB_SERVER_HOST, run_pid=p.pid, config=config, cmd=process_cmd) - return {"run_pid": p.pid, "worker_id": worker_id, "cmd": process_cmd} - - @classmethod - def generate_common_cmd(cls, task, config_dir, config, log_dir, worker_id): - session_id = job_utils.generate_session_id(task.f_task_id, task.f_task_version, task.f_role, task.f_party_id) - federation_session_id = job_utils.generate_task_version_id(task.f_task_id, task.f_task_version) - config_path, result_path = cls.get_config(config_dir=config_dir, config=config, log_dir=log_dir) - cmd = [ - "--job_id", task.f_job_id, - "--component_name", task.f_component_name, - "--task_id", task.f_task_id, - "--task_version", task.f_task_version, - "--role", task.f_role, - "--party_id", task.f_party_id, - "--config", config_path, - '--result', result_path, - "--log_dir", log_dir, - "--parent_log_dir", os.path.dirname(log_dir), - "--worker_id", worker_id, - "--run_ip", RuntimeConfig.JOB_SERVER_HOST, - "--run_port", RuntimeConfig.HTTP_PORT, - "--job_server", f"{RuntimeConfig.JOB_SERVER_HOST}:{RuntimeConfig.HTTP_PORT}", - "--session_id", session_id, - "--federation_session_id", federation_session_id - ] - return session_id, federation_session_id, cmd - - @classmethod - def get_process_dirs(cls, worker_name: WorkerName, job_id=None, role=None, party_id=None, task: Task = None): - worker_id = uuid1().hex - party_id = str(party_id) - if task: - config_dir = job_utils.get_job_directory(job_id, role, party_id, task.f_component_name, task.f_task_id, - str(task.f_task_version), worker_name.value, worker_id) - log_dir = job_utils.get_job_log_directory(job_id, role, party_id, task.f_component_name) - elif job_id and role and party_id: - config_dir = job_utils.get_job_directory(job_id, role, party_id, worker_name.value, worker_id) - log_dir = job_utils.get_job_log_directory(job_id, role, party_id, worker_name.value, worker_id) - else: - config_dir = job_utils.get_general_worker_directory(worker_name.value, worker_id) - log_dir = job_utils.get_general_worker_log_directory(worker_name.value, worker_id) - os.makedirs(config_dir, exist_ok=True) - return worker_id, config_dir, log_dir - - @classmethod - def get_config(cls, config_dir, config, log_dir): - config_path = os.path.join(config_dir, "config.json") - with open(config_path, 'w') as fw: - fw.write(json_dumps(config)) - result_path = os.path.join(config_dir, "result.json") - return config_path, result_path - - @classmethod - def get_env(cls, job_id, provider_info): - provider = ComponentProvider(**provider_info) - env = provider.env.copy() - env["PYTHONPATH"] = os.path.dirname(provider.path) - if job_id: - env["FATE_JOB_ID"] = job_id - return env - - @classmethod - def cmd_to_func_kwargs(cls, cmd): - kwargs = {} - for i in range(2, len(cmd), 2): - kwargs[cmd[i].lstrip("--")] = cmd[i+1] - return kwargs - - @classmethod - @DB.connection_context() - def save_worker_info(cls, task: Task, worker_name: WorkerName, worker_id, **kwargs): - worker = WorkerInfo() - ignore_attr = auto_date_timestamp_db_field() - for attr, value in task.to_dict().items(): - if hasattr(worker, attr) and attr not in ignore_attr and value is not None: - setattr(worker, attr, value) - worker.f_create_time = current_timestamp() - worker.f_worker_name = worker_name.value - worker.f_worker_id = worker_id - for k, v in kwargs.items(): - attr = f"f_{k}" - if hasattr(worker, attr) and v is not None: - setattr(worker, attr, v) - rows = worker.save(force_insert=True) - if rows != 1: - raise Exception("save worker info failed") - - @classmethod - @DB.connection_context() - def kill_task_all_workers(cls, task: Task): - schedule_logger(task.f_job_id).info(start_log("kill all workers", task=task)) - workers_info = WorkerInfo.query(task_id=task.f_task_id, task_version=task.f_task_version, role=task.f_role, - party_id=task.f_party_id) - for worker_info in workers_info: - schedule_logger(task.f_job_id).info( - start_log(f"kill {worker_info.f_worker_name}({worker_info.f_run_pid})", task=task)) - try: - cls.kill_worker(worker_info) - schedule_logger(task.f_job_id).info( - successful_log(f"kill {worker_info.f_worker_name}({worker_info.f_run_pid})", task=task)) - except Exception as e: - schedule_logger(task.f_job_id).warning( - failed_log(f"kill {worker_info.f_worker_name}({worker_info.f_run_pid})", task=task), exc_info=True) - schedule_logger(task.f_job_id).info(successful_log("kill all workers", task=task)) - - @classmethod - def kill_worker(cls, worker_info: WorkerInfo): - process_utils.kill_process(pid=worker_info.f_run_pid, expected_cmdline=worker_info.f_cmd) diff --git a/python/fate_flow/model/__init__.py b/python/fate_flow/model/__init__.py deleted file mode 100644 index 83576b0a8..000000000 --- a/python/fate_flow/model/__init__.py +++ /dev/null @@ -1,138 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import importlib -import inspect -from functools import wraps -from pathlib import Path - -from filelock import FileLock as _FileLock - -from fate_arch.protobuf.python.default_empty_fill_pb2 import DefaultEmptyFillMessage - -from fate_flow.component_env_utils import provider_utils -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.settings import stat_logger - - -def serialize_buffer_object(buffer_object): - # the type is bytes, not str - serialized_string = buffer_object.SerializeToString() - if not serialized_string: - fill_message = DefaultEmptyFillMessage() - fill_message.flag = 'set' - serialized_string = fill_message.SerializeToString() - return serialized_string - - -def get_proto_buffer_class(buffer_name): - module_path, base_import_path = provider_utils.get_provider_model_paths(RuntimeConfig.COMPONENT_PROVIDER) - exception = ModuleNotFoundError(f'no module named {buffer_name}') - for f in module_path.glob('*.py'): - try: - proto_module = importlib.import_module('.'.join([*base_import_path, f.stem])) - for name, obj in inspect.getmembers(proto_module): - if inspect.isclass(obj) and name == buffer_name: - return obj - except Exception as e: - exception = e - stat_logger.warning(e) - raise exception - - -def parse_proto_object(buffer_name, serialized_string, buffer_class=None): - try: - if buffer_class is None: - buffer_class = get_proto_buffer_class(buffer_name) - buffer_object = buffer_class() - except Exception as e: - stat_logger.exception('Can not restore proto buffer object') - raise e - buffer_name = type(buffer_object).__name__ - - try: - buffer_object.ParseFromString(serialized_string) - except Exception as e1: - stat_logger.exception(e1) - try: - DefaultEmptyFillMessage().ParseFromString(serialized_string) - buffer_object.ParseFromString(bytes()) - except Exception as e2: - stat_logger.exception(e2) - raise e1 - else: - stat_logger.info(f'parse {buffer_name} proto object with default values') - else: - stat_logger.info(f'parse {buffer_name} proto object normal') - return buffer_object - - -def lock(method): - @wraps(method) - def magic(self, *args, **kwargs): - with self.lock: - return method(self, *args, **kwargs) - return magic - - -def local_cache_required(locking=False): - def decorator(method): - @wraps(method) - def magic(self, *args, **kwargs): - if not self.exists(): - raise FileNotFoundError(f'Can not found {self.model_id} {self.model_version} model local cache') - if not locking: - return method(self, *args, **kwargs) - with self.lock: - return method(self, *args, **kwargs) - return magic - return decorator - - -class Locker: - - def __init__(self, directory): - if isinstance(directory, str): - directory = Path(directory) - self.directory = directory - self.lock = self._lock - - @property - def _lock(self): - return FileLock(self.directory / '.lock') - - def __copy__(self): - return self - - def __deepcopy__(self, memo): - return self - - # https://docs.python.org/3/library/pickle.html#handling-stateful-objects - def __getstate__(self): - state = self.__dict__.copy() - state.pop('lock') - return state - - def __setstate__(self, state): - self.__dict__.update(state) - self.lock = self._lock - - -class FileLock(_FileLock): - - def _acquire(self, *args, **kwargs): - Path(self._lock_file).parent.mkdir(parents=True, exist_ok=True) - - super()._acquire(*args, **kwargs) diff --git a/python/fate_flow/model/checkpoint.py b/python/fate_flow/model/checkpoint.py deleted file mode 100644 index 9f874972b..000000000 --- a/python/fate_flow/model/checkpoint.py +++ /dev/null @@ -1,276 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import hashlib -from pathlib import Path -from typing import Dict, Tuple -from shutil import copytree, rmtree -from base64 import b64encode -from datetime import datetime -from collections import deque, OrderedDict - -from ruamel import yaml - -from fate_arch.common.base_utils import json_dumps, json_loads - -from fate_flow.settings import stat_logger -from fate_flow.entity import RunParameters -from fate_flow.utils.model_utils import gen_party_model_id -from fate_flow.utils.base_utils import get_fate_flow_directory -from fate_flow.model import Locker - - -class Checkpoint(Locker): - - def __init__(self, directory: Path, step_index: int, step_name: str): - self.step_index = step_index - self.step_name = step_name - self.create_time = None - - directory = directory / f'{step_index}#{step_name}' - self.database = directory / 'database.yaml' - - super().__init__(directory) - - @property - def available(self): - return self.database.exists() - - def save(self, model_buffers: Dict[str, Tuple[str, bytes, dict]]): - if not model_buffers: - raise ValueError('model_buffers is empty.') - - self.create_time = datetime.utcnow() - data = { - 'step_index': self.step_index, - 'step_name': self.step_name, - 'create_time': self.create_time.isoformat(), - 'models': {}, - } - - model_data = {} - for model_name, (pb_name, serialized_string, json_format_dict) in model_buffers.items(): - model_data[model_name] = (serialized_string, json_format_dict) - - data['models'][model_name] = { - 'sha1': hashlib.sha1(serialized_string).hexdigest(), - 'buffer_name': pb_name, - } - - self.directory.mkdir(parents=True, exist_ok=True) - - with self.lock: - for model_name, model in data['models'].items(): - serialized_string, json_format_dict = model_data[model_name] - (self.directory / f'{model_name}.pb').write_bytes(serialized_string) - (self.directory / f'{model_name}.json').write_text(json_dumps(json_format_dict), 'utf8') - - self.database.write_text(yaml.dump(data, Dumper=yaml.RoundTripDumper), 'utf8') - - stat_logger.info(f'Checkpoint saved. path: {self.directory}') - return self.directory - - def read_database(self): - with self.lock: - data = yaml.safe_load(self.database.read_text('utf8')) - if data['step_index'] != self.step_index or data['step_name'] != self.step_name: - raise ValueError('Checkpoint may be incorrect: step_index or step_name dose not match. ' - f'filepath: {self.database} ' - f'expected step_index: {self.step_index} actual step_index: {data["step_index"]} ' - f'expected step_name: {self.step_name} actual step_index: {data["step_name"]}') - - self.create_time = datetime.fromisoformat(data['create_time']) - return data - - def read(self, parse_models: bool = True, include_database: bool = False): - data = self.read_database() - - with self.lock: - for model_name, model in data['models'].items(): - model['filepath_pb'] = self.directory / f'{model_name}.pb' - model['filepath_json'] = self.directory / f'{model_name}.json' - if not model['filepath_pb'].exists() or not model['filepath_json'].exists(): - raise FileNotFoundError( - 'Checkpoint is incorrect: protobuf file or json file not found. ' - f'protobuf filepath: {model["filepath_pb"]} json filepath: {model["filepath_json"]}' - ) - - model_data = { - model_name: ( - model['filepath_pb'].read_bytes(), - json_loads(model['filepath_json'].read_text('utf8')), - ) - for model_name, model in data['models'].items() - } - - for model_name, model in data['models'].items(): - serialized_string, json_format_dict = model_data[model_name] - - sha1 = hashlib.sha1(serialized_string).hexdigest() - if sha1 != model['sha1']: - raise ValueError('Checkpoint may be incorrect: hash dose not match. ' - f'filepath: {model["filepath"]} expected: {model["sha1"]} actual: {sha1}') - - data['models'] = { - model_name: ( - model['buffer_name'], - *model_data[model_name], - ) if parse_models - else b64encode(model_data[model_name][0]).decode('ascii') - for model_name, model in data['models'].items() - } - return data if include_database else data['models'] - - def remove(self): - self.create_time = None - rmtree(self.directory) - - def to_dict(self, include_models: bool = False): - if not include_models: - return self.read_database() - return self.read(False, True) - - -class CheckpointManager: - - def __init__(self, job_id: str = None, role: str = None, party_id: int = None, - model_id: str = None, model_version: str = None, - component_name: str = None, component_module_name: str = None, - task_id: str = None, task_version: int = None, - job_parameters: RunParameters = None, - max_to_keep: int = None - ): - self.job_id = job_id - self.role = role - self.party_id = party_id - self.model_id = model_id - self.model_version = model_version - self.party_model_id = gen_party_model_id(self.model_id, self.role, self.party_id) - self.component_name = component_name if component_name else 'pipeline' - self.module_name = component_module_name if component_module_name else 'Pipeline' - self.task_id = task_id - self.task_version = task_version - self.job_parameters = job_parameters - - self.directory = (Path(get_fate_flow_directory()) / 'model_local_cache' / - self.party_model_id / model_version / 'checkpoint' / self.component_name) - - if isinstance(max_to_keep, int): - if max_to_keep <= 0: - raise ValueError('max_to_keep must be positive') - elif max_to_keep is not None: - raise TypeError('max_to_keep must be an integer') - self.checkpoints = deque(maxlen=max_to_keep) - - def load_checkpoints_from_disk(self): - checkpoints = [] - for directory in self.directory.glob('*'): - if not directory.is_dir() or '#' not in directory.name: - continue - - step_index, step_name = directory.name.split('#', 1) - checkpoint = Checkpoint(self.directory, int(step_index), step_name) - - if not checkpoint.available: - continue - checkpoints.append(checkpoint) - - self.checkpoints = deque(sorted(checkpoints, key=lambda i: i.step_index), self.max_checkpoints_number) - - @property - def checkpoints_number(self): - return len(self.checkpoints) - - @property - def max_checkpoints_number(self): - return self.checkpoints.maxlen - - @property - def number_indexed_checkpoints(self): - return OrderedDict((i.step_index, i) for i in self.checkpoints) - - @property - def name_indexed_checkpoints(self): - return OrderedDict((i.step_name, i) for i in self.checkpoints) - - def get_checkpoint_by_index(self, step_index: int): - return self.number_indexed_checkpoints.get(step_index) - - def get_checkpoint_by_name(self, step_name: str): - return self.name_indexed_checkpoints.get(step_name) - - @property - def latest_checkpoint(self): - if self.checkpoints: - return self.checkpoints[-1] - - @property - def latest_step_index(self): - if self.latest_checkpoint is not None: - return self.latest_checkpoint.step_index - - @property - def latest_step_name(self): - if self.latest_checkpoint is not None: - return self.latest_checkpoint.step_name - - def new_checkpoint(self, step_index: int, step_name: str): - if self.job_parameters is not None and self.job_parameters.job_type == 'predict': - raise ValueError('Cannot create checkpoint in predict job.') - - popped_checkpoint = None - if self.max_checkpoints_number and self.checkpoints_number >= self.max_checkpoints_number: - popped_checkpoint = self.checkpoints[0] - - checkpoint = Checkpoint(self.directory, step_index, step_name) - self.checkpoints.append(checkpoint) - - if popped_checkpoint is not None: - popped_checkpoint.remove() - - return checkpoint - - def clean(self): - self.checkpoints = deque(maxlen=self.max_checkpoints_number) - rmtree(self.directory) - - # copy the checkpoint as a component model to the new model version - def deploy(self, new_model_version: str, model_alias: str, step_index: int = None, step_name: str = None): - if step_index is not None: - checkpoint = self.get_checkpoint_by_index(step_index) - elif step_name is not None: - checkpoint = self.get_checkpoint_by_name(step_name) - else: - raise KeyError('step_index or step_name is required.') - - if checkpoint is None: - raise TypeError('Checkpoint not found.') - # check files hash - checkpoint.read() - - directory = Path(get_fate_flow_directory()) / 'model_local_cache' / self.party_model_id / new_model_version - target = directory / 'variables' / 'data' / self.component_name / model_alias - locker = Locker(directory) - - with locker.lock: - rmtree(target, True) - copytree(checkpoint.directory, target, - ignore=lambda src, names: {i for i in names if i.startswith('.')}) - - for f in target.glob('*.pb'): - f.replace(f.with_suffix('')) - - def to_dict(self, include_models: bool = False): - return [checkpoint.to_dict(include_models) for checkpoint in self.checkpoints] diff --git a/python/fate_flow/model/model_storage_base.py b/python/fate_flow/model/model_storage_base.py deleted file mode 100644 index 1f2cf9830..000000000 --- a/python/fate_flow/model/model_storage_base.py +++ /dev/null @@ -1,68 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from abc import ABC, abstractmethod -from contextlib import AbstractContextManager - - -class ModelStorageBase(ABC): - - @abstractmethod - def exists(self, model_id: str, model_version: str, store_address: dict): - pass - - @abstractmethod - def store(self, model_id: str, model_version: str, store_address: dict, force_update: bool = False): - """ - Store the model from local cache to a reliable system - :param model_id: - :param model_version: - :param store_address: - :return: - """ - pass - - @abstractmethod - def restore(self, model_id: str, model_version: str, store_address: dict, force_update: bool = False, hash_: str = None): - """ - Restore model from storage system to local cache - :param model_id: - :param model_version: - :param store_address: - :return: - """ - pass - - -class ComponentStorageBase(AbstractContextManager): - - def __exit__(self, *exc): - pass - - @abstractmethod - def exists(self, party_model_id, model_version, component_name): - pass - - @abstractmethod - def upload(self, party_model_id, model_version, component_name): - pass - - @abstractmethod - def download(self, party_model_id, model_version, component_name, hash_=None): - pass - - @abstractmethod - def copy(self, party_model_id, model_version, component_name, source_model_version): - pass diff --git a/python/fate_flow/model/mysql_model_storage.py b/python/fate_flow/model/mysql_model_storage.py deleted file mode 100644 index 80faabcbb..000000000 --- a/python/fate_flow/model/mysql_model_storage.py +++ /dev/null @@ -1,330 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from re import I -import sys -from copy import deepcopy - -from peewee import ( - BigIntegerField, CharField, CompositeKey, - IntegerField, PeeweeException, Value, -) -from playhouse.pool import PooledMySQLDatabase - -from fate_arch.common.base_utils import ( - current_timestamp, deserialize_b64, - serialize_b64, timestamp_to_date, -) -from fate_arch.common.conf_utils import decrypt_database_password, decrypt_database_config -from fate_arch.metastore.base_model import LongTextField - -from fate_flow.db.db_models import DataBaseModel -from fate_flow.model.model_storage_base import ComponentStorageBase, ModelStorageBase -from fate_flow.pipelined_model.pipelined_model import PipelinedModel -from fate_flow.pipelined_model.pipelined_component import PipelinedComponent -from fate_flow.utils.log_utils import getLogger - - -LOGGER = getLogger() -DB = PooledMySQLDatabase(None) - -SLICE_MAX_SIZE = 1024*1024*8 - - -class MysqlModelStorage(ModelStorageBase): - - def exists(self, model_id: str, model_version: str, store_address: dict): - self.get_connection(store_address) - - try: - with DB.connection_context(): - counts = MachineLearningModel.select().where( - MachineLearningModel.f_model_id == model_id, - MachineLearningModel.f_model_version == model_version, - ).count() - return counts > 0 - except PeeweeException as e: - # Table doesn't exist - if e.args and e.args[0] == 1146: - return False - - raise e - finally: - self.close_connection() - - def store(self, model_id: str, model_version: str, store_address: dict, force_update: bool = False): - ''' - Store the model from local cache to mysql - :param model_id: - :param model_version: - :param store_address: - :param force_update: - :return: - ''' - if not force_update and self.exists(model_id, model_version, store_address): - raise FileExistsError(f'The model {model_id} {model_version} already exists in the database.') - - try: - self.get_connection(store_address) - DB.create_tables([MachineLearningModel]) - - model = PipelinedModel(model_id, model_version) - hash_ = model.packaging_model() - - with open(model.archive_model_file_path, 'rb') as fr, DB.connection_context(): - MachineLearningModel.delete().where( - MachineLearningModel.f_model_id == model_id, - MachineLearningModel.f_model_version == model_version, - ).execute() - - LOGGER.info(f'Starting store model {model_id} {model_version}.') - - slice_index = 0 - while True: - content = fr.read(SLICE_MAX_SIZE) - if not content: - break - - model_in_table = MachineLearningModel() - model_in_table.f_model_id = model_id - model_in_table.f_model_version = model_version - model_in_table.f_content = serialize_b64(content, to_str=True) - model_in_table.f_size = sys.getsizeof(model_in_table.f_content) - model_in_table.f_slice_index = slice_index - - rows = model_in_table.save(force_insert=True) - if not rows: - raise IndexError(f'Save slice index {slice_index} failed') - - LOGGER.info(f'Saved slice index {slice_index} of model {model_id} {model_version}.') - slice_index += 1 - except Exception as e: - LOGGER.exception(e) - raise Exception(f'Store model {model_id} {model_version} to mysql failed.') - else: - LOGGER.info(f'Store model {model_id} {model_version} to mysql successfully.') - return hash_ - finally: - self.close_connection() - - def restore(self, model_id: str, model_version: str, store_address: dict, force_update: bool = False, hash_: str = None): - ''' - Restore model from mysql to local cache - :param model_id: - :param model_version: - :param store_address: - :return: - ''' - model = PipelinedModel(model_id, model_version) - self.get_connection(store_address) - - try: - with DB.connection_context(): - models_in_tables = MachineLearningModel.select().where( - MachineLearningModel.f_model_id == model_id, - MachineLearningModel.f_model_version == model_version, - ).order_by(MachineLearningModel.f_slice_index) - - with open(model.archive_model_file_path, 'wb') as fw: - for models_in_table in models_in_tables: - fw.write(deserialize_b64(models_in_table.f_content)) - - if fw.tell() == 0: - raise IndexError(f'Cannot found model in table.') - - model.unpack_model(model.archive_model_file_path, force_update, hash_) - except Exception as e: - LOGGER.exception(e) - raise Exception(f'Restore model {model_id} {model_version} from mysql failed.') - else: - LOGGER.info(f'Restore model to {model.archive_model_file_path} from mysql successfully.') - finally: - self.close_connection() - - @staticmethod - def get_connection(store_address: dict): - store_address = deepcopy(store_address) - store_address.pop('storage', None) - database = store_address.pop('database') - - store_address = decrypt_database_config(store_address, 'password') - DB.init(database, **store_address) - - @staticmethod - def close_connection(): - if DB: - try: - DB.close() - except Exception as e: - LOGGER.exception(e) - - -class MysqlComponentStorage(ComponentStorageBase): - - def __init__(self, database, user, password, host, port, **connect_kwargs): - self.database = database - self.user = user - self.password = decrypt_database_password(password) - self.host = host - self.port = port - self.connect_kwargs = connect_kwargs - - def __enter__(self): - DB.init(self.database, user=self.user, password=self.password, host=self.host, port=self.port, **self.connect_kwargs) - - return self - - def __exit__(self, *exc): - DB.close() - - def exists(self, party_model_id, model_version, component_name): - try: - with DB.connection_context(): - counts = MachineLearningComponent.select().where( - MachineLearningComponent.f_party_model_id == party_model_id, - MachineLearningComponent.f_model_version == model_version, - MachineLearningComponent.f_component_name == component_name, - ).count() - return counts > 0 - except PeeweeException as e: - # Table doesn't exist - if e.args and e.args[0] == 1146: - return False - - raise e - - def upload(self, party_model_id, model_version, component_name): - DB.create_tables([MachineLearningComponent]) - - pipelined_component = PipelinedComponent(party_model_id=party_model_id, model_version=model_version) - filename, hash_ = pipelined_component.pack_component(component_name) - - with open(filename, 'rb') as fr, DB.connection_context(): - MachineLearningComponent.delete().where( - MachineLearningComponent.f_party_model_id == party_model_id, - MachineLearningComponent.f_model_version == model_version, - MachineLearningComponent.f_component_name == component_name, - ).execute() - - slice_index = 0 - while True: - content = fr.read(SLICE_MAX_SIZE) - if not content: - break - - model_in_table = MachineLearningComponent() - model_in_table.f_party_model_id = party_model_id - model_in_table.f_model_version = model_version - model_in_table.f_component_name = component_name - model_in_table.f_content = serialize_b64(content, to_str=True) - model_in_table.f_size = sys.getsizeof(model_in_table.f_content) - model_in_table.f_slice_index = slice_index - - rows = model_in_table.save(force_insert=True) - if not rows: - raise IndexError(f'Save slice index {slice_index} failed') - - slice_index += 1 - - return hash_ - - def download(self, party_model_id, model_version, component_name, hash_=None): - with DB.connection_context(): - models_in_tables = MachineLearningComponent.select().where( - MachineLearningComponent.f_party_model_id == party_model_id, - MachineLearningComponent.f_model_version == model_version, - MachineLearningComponent.f_component_name == component_name, - ).order_by(MachineLearningComponent.f_slice_index) - - pipelined_component = PipelinedComponent(party_model_id=party_model_id, model_version=model_version) - - with open(pipelined_component.get_archive_path(component_name), 'wb') as fw: - for models_in_table in models_in_tables: - fw.write(deserialize_b64(models_in_table.f_content)) - - if fw.tell() == 0: - raise IndexError(f'Cannot found component model in table.') - - pipelined_component.unpack_component(component_name, hash_) - - @DB.connection_context() - def copy(self, party_model_id, model_version, component_name, source_model_version): - now = current_timestamp() - - source = MachineLearningComponent.select( - MachineLearningComponent.f_create_time, - MachineLearningComponent.f_create_date, - Value(now).alias('f_update_time'), - Value(timestamp_to_date(now)).alias('f_update_date'), - - MachineLearningComponent.f_party_model_id, - Value(model_version).alias('f_model_version'), - MachineLearningComponent.f_component_name, - - MachineLearningComponent.f_size, - MachineLearningComponent.f_content, - MachineLearningComponent.f_slice_index, - ).where( - MachineLearningComponent.f_party_model_id == party_model_id, - MachineLearningComponent.f_model_version == source_model_version, - MachineLearningComponent.f_component_name == component_name, - ).order_by(MachineLearningComponent.f_slice_index) - - rows = MachineLearningComponent.insert_from(source, ( - MachineLearningComponent.f_create_time, - MachineLearningComponent.f_create_date, - MachineLearningComponent.f_update_time, - MachineLearningComponent.f_update_date, - - MachineLearningComponent.f_party_model_id, - MachineLearningComponent.f_model_version, - MachineLearningComponent.f_component_name, - - MachineLearningComponent.f_size, - MachineLearningComponent.f_content, - MachineLearningComponent.f_slice_index, - )).execute() - - if not rows: - raise IndexError(f'Copy component model failed.') - - -class MachineLearningModel(DataBaseModel): - f_model_id = CharField(max_length=100, index=True) - f_model_version = CharField(max_length=100, index=True) - f_size = BigIntegerField(default=0) - f_content = LongTextField(default='') - f_slice_index = IntegerField(default=0, index=True) - - class Meta: - database = DB - db_table = 't_machine_learning_model' - primary_key = CompositeKey('f_model_id', 'f_model_version', 'f_slice_index') - - -class MachineLearningComponent(DataBaseModel): - f_party_model_id = CharField(max_length=100, index=True) - f_model_version = CharField(max_length=100, index=True) - f_component_name = CharField(max_length=100, index=True) - f_size = BigIntegerField(default=0) - f_content = LongTextField(default='') - f_slice_index = IntegerField(default=0, index=True) - - class Meta: - database = DB - db_table = 't_machine_learning_component' - indexes = ( - (('f_party_model_id', 'f_model_version', 'f_component_name', 'f_slice_index'), True), - ) diff --git a/python/fate_flow/model/sync_model.py b/python/fate_flow/model/sync_model.py deleted file mode 100644 index 8de41b932..000000000 --- a/python/fate_flow/model/sync_model.py +++ /dev/null @@ -1,227 +0,0 @@ -# -# Copyright 2022 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from copy import deepcopy -from hashlib import sha256 -from typing import Tuple - -from peewee import DoesNotExist - -from fate_flow.db.db_models import ( - DB, PipelineComponentMeta, - MachineLearningModelInfo as MLModel, -) -from fate_flow.db.service_registry import ServerRegistry -from fate_flow.model import ( - lock, model_storage_base, - mysql_model_storage, tencent_cos_model_storage, -) -from fate_flow.pipelined_model import Pipelined -from fate_flow.pipelined_model.pipelined_model import PipelinedModel -from fate_flow.settings import HOST - - -model_storage_map = { - 'mysql': mysql_model_storage.MysqlModelStorage, - 'tencent_cos': tencent_cos_model_storage.TencentCOSModelStorage, -} - -component_storage_map = { - 'mysql': mysql_model_storage.MysqlComponentStorage, - 'tencent_cos': tencent_cos_model_storage.TencentCOSComponentStorage, -} - - -def get_storage(storage_map: dict) -> Tuple[model_storage_base.ModelStorageBase, dict]: - store_address = deepcopy(ServerRegistry.MODEL_STORE_ADDRESS) - - store_type = store_address.pop('storage') - if store_type not in storage_map: - raise KeyError(f"Model storage '{store_type}' is not supported.") - - return storage_map[store_type], store_address - - -class SyncModel(Pipelined): - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - self.pipelined_model = PipelinedModel(self.party_model_id, self.model_version) - - storage, storage_address = get_storage(model_storage_map) - self.model_storage = storage() - self.model_storage_parameters = { - 'model_id': self.party_model_id, - 'model_version': self.model_version, - 'store_address': storage_address, - } - - self.lock = DB.lock( - sha256( - '_'.join(( - 'sync_model', - self.party_model_id, - self.model_version, - )).encode('utf-8') - ).hexdigest(), - -1, - ) - - @DB.connection_context() - def db_exists(self): - try: - self.get_model() - except DoesNotExist: - return False - else: - return True - - def local_exists(self): - return self.pipelined_model.exists() - - def remote_exists(self): - return self.model_storage.exists(**self.model_storage_parameters) - - def get_model(self): - return MLModel.get( - MLModel.f_role == self.role, - MLModel.f_party_id == self.party_id, - MLModel.f_model_id == self.model_id, - MLModel.f_model_version == self.model_version, - ) - - @DB.connection_context() - def upload(self, force_update=False): - if self.remote_exists() and not force_update: - return - - with self.lock: - model = self.get_model() - - hash_ = self.model_storage.store( - force_update=force_update, - **self.model_storage_parameters, - ) - - model.f_archive_sha256 = hash_ - model.f_archive_from_ip = HOST - model.save() - - return model - - @DB.connection_context() - def download(self, force_update=False): - if self.local_exists() and not force_update: - return - - with self.lock: - model = self.get_model() - - self.model_storage.restore( - force_update=force_update, hash_=model.f_archive_sha256, - **self.model_storage_parameters, - ) - - return model - - -class SyncComponent(Pipelined): - - def __init__(self, *, component_name, **kwargs): - super().__init__(**kwargs) - self.component_name = component_name - - self.pipelined_model = PipelinedModel(self.party_model_id, self.model_version) - - storage, storage_address = get_storage(component_storage_map) - self.component_storage = storage(**storage_address) - self.component_storage_parameters = ( - self.party_model_id, - self.model_version, - self.component_name, - ) - - self.query_args = ( - PipelineComponentMeta.f_role == self.role, - PipelineComponentMeta.f_party_id == self.party_id, - PipelineComponentMeta.f_model_id == self.model_id, - PipelineComponentMeta.f_model_version == self.model_version, - PipelineComponentMeta.f_component_name == self.component_name, - ) - - self.lock = DB.lock( - sha256( - '_'.join(( - 'sync_component', - self.party_model_id, - self.model_version, - self.component_name, - )).encode('utf-8') - ).hexdigest(), - -1, - ) - - @DB.connection_context() - def db_exists(self): - return PipelineComponentMeta.select().where(*self.query_args).count() > 0 - - def local_exists(self): - return self.pipelined_model.pipelined_component.exists(self.component_name) - - def remote_exists(self): - with self.component_storage as storage: - return storage.exists(*self.component_storage_parameters) - - def get_archive_hash(self): - query = tuple(PipelineComponentMeta.select().where(*self.query_args).group_by( - PipelineComponentMeta.f_archive_sha256, PipelineComponentMeta.f_archive_from_ip)) - if len(query) != 1: - raise ValueError(f'The define_meta data of {self.component_name} in database is invalid.') - - return query[0].f_archive_sha256 - - def update_archive_hash(self, hash_): - PipelineComponentMeta.update( - f_archive_sha256=hash_, - f_archive_from_ip=HOST, - ).where(*self.query_args).execute() - - @DB.connection_context() - @lock - def upload(self): - # check the data in database - self.get_archive_hash() - - with self.component_storage as storage: - hash_ = storage.upload(*self.component_storage_parameters) - - self.update_archive_hash(hash_) - - @DB.connection_context() - @lock - def download(self): - hash_ = self.get_archive_hash() - - with self.component_storage as storage: - storage.download(*self.component_storage_parameters, hash_) - - @DB.connection_context() - @lock - def copy(self, source_model_version, hash_): - with self.component_storage as storage: - storage.copy(*self.component_storage_parameters, source_model_version) - - self.update_archive_hash(hash_) diff --git a/python/fate_flow/model/tencent_cos_model_storage.py b/python/fate_flow/model/tencent_cos_model_storage.py deleted file mode 100644 index 251b5bc45..000000000 --- a/python/fate_flow/model/tencent_cos_model_storage.py +++ /dev/null @@ -1,178 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from copy import deepcopy - -from qcloud_cos import CosConfig, CosS3Client -from qcloud_cos.cos_exception import CosServiceError - -from fate_flow.model.model_storage_base import ComponentStorageBase, ModelStorageBase -from fate_flow.pipelined_model.pipelined_model import PipelinedModel -from fate_flow.pipelined_model.pipelined_component import PipelinedComponent -from fate_flow.utils.log_utils import getLogger - - -LOGGER = getLogger() - - -class TencentCOSModelStorage(ModelStorageBase): - - def store_key(self, model_id: str, model_version: str): - return f'FATEFlow/PipelinedModel/{model_id}/{model_version}.zip' - - def exists(self, model_id: str, model_version: str, store_address: dict): - store_key = self.store_key(model_id, model_version) - cos = self.get_connection(store_address) - - try: - cos.head_object( - Bucket=store_address["Bucket"], - Key=store_key, - ) - except CosServiceError as e: - if e.get_error_code() != 'NoSuchResource': - raise e - return False - else: - return True - - def store(self, model_id: str, model_version: str, store_address: dict, force_update: bool = False): - """ - Store the model from local cache to cos - :param model_id: - :param model_version: - :param store_address: - :param force_update: - :return: - """ - store_key = self.store_key(model_id, model_version) - if not force_update and self.exists(model_id, model_version, store_address): - raise FileExistsError(f"The object {store_key} already exists.") - - model = PipelinedModel(model_id, model_version) - cos = self.get_connection(store_address) - - try: - hash_ = model.packaging_model() - - response = cos.upload_file( - Bucket=store_address["Bucket"], - LocalFilePath=model.archive_model_file_path, - Key=store_key, - EnableMD5=True, - ) - except Exception as e: - LOGGER.exception(e) - raise Exception(f"Store model {model_id} {model_version} to Tencent COS failed.") - else: - LOGGER.info(f"Store model {model_id} {model_version} to Tencent COS successfully. " - f"Archive path: {model.archive_model_file_path} Key: {store_key} ETag: {response['ETag']}") - return hash_ - - def restore(self, model_id: str, model_version: str, store_address: dict, force_update: bool = False, hash_: str = None): - """ - Restore model from cos to local cache - :param model_id: - :param model_version: - :param store_address: - :return: - """ - store_key = self.store_key(model_id, model_version) - model = PipelinedModel(model_id, model_version) - cos = self.get_connection(store_address) - - try: - cos.download_file( - Bucket=store_address["Bucket"], - Key=store_key, - DestFilePath=model.archive_model_file_path, - EnableCRC=True, - ) - - model.unpack_model(model.archive_model_file_path, force_update, hash_) - except Exception as e: - LOGGER.exception(e) - raise Exception(f"Restore model {model_id} {model_version} from Tencent COS failed.") - else: - LOGGER.info(f"Restore model {model_id} {model_version} from Tencent COS successfully. " - f"Archive path: {model.archive_model_file_path} Key: {store_key}") - - @staticmethod - def get_connection(store_address: dict): - store_address = deepcopy(store_address) - store_address.pop('storage', None) - store_address.pop('Bucket') - - return CosS3Client(CosConfig(**store_address)) - - -class TencentCOSComponentStorage(ComponentStorageBase): - - def __init__(self, Region, SecretId, SecretKey, Bucket): - self.client = CosS3Client(CosConfig(Region=Region, SecretId=SecretId, SecretKey=SecretKey)) - self.bucket = Bucket - self.region = Region - - def get_key(self, party_model_id, model_version, component_name): - return f'FATEFlow/PipelinedComponent/{party_model_id}/{model_version}/{component_name}.zip' - - def exists(self, party_model_id, model_version, component_name): - key = self.get_key(party_model_id, model_version, component_name) - - try: - self.client.head_object( - Bucket=self.bucket, - Key=key, - ) - except CosServiceError as e: - if e.get_error_code() != 'NoSuchResource': - raise e - return False - else: - return True - - def upload(self, party_model_id, model_version, component_name): - pipelined_component = PipelinedComponent(party_model_id=party_model_id, model_version=model_version) - filename, hash_ = pipelined_component.pack_component(component_name) - - self.client.upload_file( - Bucket=self.bucket, - LocalFilePath=filename, - Key=self.get_key(party_model_id, model_version, component_name), - EnableMD5=True, - ) - return hash_ - - def download(self, party_model_id, model_version, component_name, hash_=None): - pipelined_component = PipelinedComponent(party_model_id=party_model_id, model_version=model_version) - - self.client.download_file( - Bucket=self.bucket, - Key=self.get_key(party_model_id, model_version, component_name), - DestFilePath=pipelined_component.get_archive_path(component_name), - EnableCRC=True, - ) - pipelined_component.unpack_component(component_name, hash_) - - def copy(self, party_model_id, model_version, component_name, source_model_version): - self.client.copy( - Bucket=self.bucket, - Key=self.get_key(party_model_id, model_version, component_name), - CopySource={ - 'Bucket': self.bucket, - 'Key': self.get_key(party_model_id, source_model_version, component_name), - 'Region': self.region, - }, - ) diff --git a/python/fate_flow/operation/job_clean.py b/python/fate_flow/operation/job_clean.py deleted file mode 100644 index cdf6b50b9..000000000 --- a/python/fate_flow/operation/job_clean.py +++ /dev/null @@ -1,68 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_flow.manager.data_manager import delete_tables_by_table_infos, delete_metric_data -from fate_flow.operation.job_tracker import Tracker -from fate_flow.operation.job_saver import JobSaver -from fate_flow.settings import stat_logger -from fate_flow.utils.job_utils import start_session_stop - - -class JobClean(object): - @classmethod - def clean_table(cls, job_id, role, party_id, component_name): - # clean data table - stat_logger.info('start delete {} {} {} {} data table'.format(job_id, role, party_id, component_name)) - - tracker = Tracker(job_id=job_id, role=role, party_id=party_id, component_name=component_name) - output_data_table_infos = tracker.get_output_data_info() - if output_data_table_infos: - delete_tables_by_table_infos(output_data_table_infos) - stat_logger.info('delete {} {} {} {} data table success'.format(job_id, role, party_id, component_name)) - - @classmethod - def start_clean_job(cls, **kwargs): - tasks = JobSaver.query_task(**kwargs) - if tasks: - for task in tasks: - try: - # clean session - stat_logger.info('start {} {} {} {} session stop'.format(task.f_job_id, task.f_role, - task.f_party_id, task.f_component_name)) - start_session_stop(task) - stat_logger.info('stop {} {} {} {} session success'.format(task.f_job_id, task.f_role, - task.f_party_id, task.f_component_name)) - except Exception as e: - stat_logger.info(f'start_session_stop occur exception: {e}') - try: - # clean data table - JobClean.clean_table(job_id=task.f_job_id, role=task.f_role, party_id=task.f_party_id, - component_name=task.f_component_name) - except Exception as e: - stat_logger.info('delete {} {} {} {} data table failed'.format(task.f_job_id, task.f_role, - task.f_party_id, - task.f_component_name)) - stat_logger.exception(e) - # according to job_id to clean metric data - try: - # clean metric data - stat_logger.info('start delete {} metric data'.format(kwargs["job_id"])) - delete_metric_data({'job_id': kwargs["job_id"]}) - stat_logger.info('delete {} metric data success'.format(kwargs["job_id"])) - except Exception as e: - stat_logger.info('delete {} metric data failed'.format(kwargs["job_id"])) - stat_logger.exception(e) - else: - raise Exception('no found task') diff --git a/python/fate_flow/operation/job_tracker.py b/python/fate_flow/operation/job_tracker.py deleted file mode 100644 index 6f67ee361..000000000 --- a/python/fate_flow/operation/job_tracker.py +++ /dev/null @@ -1,392 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import operator -import typing - -from fate_arch import session, storage -from fate_arch.abc import CTableABC -from fate_arch.common import EngineType -from fate_arch.common.base_utils import current_timestamp, deserialize_b64, serialize_b64 -from fate_arch.common.data_utils import default_output_info -from fate_arch.storage import StorageEngine - -from fate_flow.db.db_models import DB, ComponentSummary, TrackingOutputDataInfo -from fate_flow.db.db_utils import bulk_insert_into_db -from fate_flow.db.job_default_config import JobDefaultConfig -from fate_flow.entity import DataCache, Metric, MetricMeta, RunParameters -from fate_flow.manager.cache_manager import CacheManager -from fate_flow.manager.metric_manager import MetricManager -from fate_flow.pipelined_model.pipelined_model import PipelinedModel -from fate_flow.utils import job_utils, model_utils -from fate_flow.utils.log_utils import schedule_logger - - -class Tracker(object): - """ - Tracker for Job/Task/Metric - """ - METRIC_DATA_PARTITION = 48 - METRIC_LIST_PARTITION = 48 - JOB_VIEW_PARTITION = 8 - - def __init__(self, job_id: str, role: str, party_id: int, - model_id: str = None, - model_version: str = None, - component_name: str = None, - component_module_name: str = None, - task_id: str = None, - task_version: int = None, - job_parameters: RunParameters = None - ): - self.job_id = job_id - self.job_parameters = job_parameters - self.role = role - self.party_id = party_id - self.component_name = component_name if component_name else job_utils.PIPELINE_COMPONENT_NAME - self.module_name = component_module_name if component_module_name else job_utils.PIPELINE_COMPONENT_MODULE_NAME - self.task_id = task_id - self.task_version = task_version - - self.model_id = model_id - self.party_model_id = model_utils.gen_party_model_id(model_id=model_id, role=role, party_id=party_id) - self.model_version = model_version - self.pipelined_model = None - if self.party_model_id and self.model_version: - self.pipelined_model = PipelinedModel(self.party_model_id, self.model_version) - self.metric_manager = MetricManager(job_id=self.job_id, role=self.role, party_id=self.party_id, component_name=self.component_name, task_id=self.task_id, task_version=self.task_version) - - def save_metric_data(self, metric_namespace: str, metric_name: str, metrics: typing.List[Metric], job_level=False): - schedule_logger(self.job_id).info( - 'save component {} on {} {} {} {} metric data'.format(self.component_name, self.role, - self.party_id, metric_namespace, metric_name)) - kv = [] - for metric in metrics: - kv.append((metric.key, metric.value)) - self.metric_manager.insert_metrics_into_db(metric_namespace, metric_name, 1, kv, job_level) - - def get_job_metric_data(self, metric_namespace: str, metric_name: str): - return self.read_metric_data(metric_namespace=metric_namespace, metric_name=metric_name, job_level=True) - - def get_metric_data(self, metric_namespace: str, metric_name: str): - return self.read_metric_data(metric_namespace=metric_namespace, metric_name=metric_name, job_level=False) - - @DB.connection_context() - def read_metric_data(self, metric_namespace: str, metric_name: str, job_level=False): - metrics = [] - for k, v in self.metric_manager.read_metrics_from_db(metric_namespace, metric_name, 1, job_level): - metrics.append(Metric(key=k, value=v)) - return metrics - - def save_metric_meta(self, metric_namespace: str, metric_name: str, metric_meta: MetricMeta, - job_level: bool = False): - schedule_logger(self.job_id).info( - 'save component {} on {} {} {} {} metric meta'.format(self.component_name, self.role, - self.party_id, metric_namespace, metric_name)) - self.metric_manager.insert_metrics_into_db(metric_namespace, metric_name, 0, metric_meta.to_dict().items(), job_level) - - @DB.connection_context() - def get_metric_meta(self, metric_namespace: str, metric_name: str, job_level: bool = False): - kv = dict() - for k, v in self.metric_manager.read_metrics_from_db(metric_namespace, metric_name, 0, job_level): - kv[k] = v - return MetricMeta(name=kv.get('name'), metric_type=kv.get('metric_type'), extra_metas=kv) - - def log_job_view(self, view_data: dict): - self.metric_manager.insert_metrics_into_db('job', 'job_view', 2, view_data.items(), job_level=True) - - @DB.connection_context() - def get_job_view(self): - view_data = {} - for k, v in self.metric_manager.read_metrics_from_db('job', 'job_view', 2, job_level=True): - view_data[k] = v - return view_data - - def save_output_data(self, computing_table, output_storage_engine, output_storage_address=None, - output_table_namespace=None, output_table_name=None, schema=None, token=None, need_read=True): - if computing_table: - if not output_table_namespace or not output_table_name: - output_table_namespace, output_table_name = default_output_info(task_id=self.task_id, task_version=self.task_version, output_type="data") - schedule_logger(self.job_id).info( - 'persisting the component output temporary table to {} {}'.format(output_table_namespace, - output_table_name)) - - part_of_limit = JobDefaultConfig.output_data_summary_count_limit - part_of_data = [] - if need_read: - for k, v in computing_table.collect(): - part_of_data.append((k, v)) - part_of_limit -= 1 - if part_of_limit == 0: - break - - session.Session.persistent(computing_table=computing_table, - namespace=output_table_namespace, - name=output_table_name, - schema=schema, - part_of_data=part_of_data, - engine=output_storage_engine, - engine_address=output_storage_address, - token=token) - - return output_table_namespace, output_table_name - else: - schedule_logger(self.job_id).info('task id {} output data table is none'.format(self.task_id)) - return None, None - - def save_table_meta(self, meta): - schedule_logger(self.job_id).info(f'start save table meta:{meta}') - address = storage.StorageTableMeta.create_address(storage_engine=meta.get("engine"), - address_dict=meta.get("address")) - table_meta = storage.StorageTableMeta(name=meta.get("name"), namespace=meta.get("namespace"), new=True) - table_meta.set_metas(**meta) - meta["address"] = address - meta["part_of_data"] = deserialize_b64(meta["part_of_data"]) - meta["schema"] = deserialize_b64(meta["schema"]) - table_meta.create() - schedule_logger(self.job_id).info(f'save table meta success') - - def get_table_meta(self, table_info): - schedule_logger(self.job_id).info(f'start get table meta:{table_info}') - table_meta_dict = storage.StorageTableMeta(namespace=table_info.get("namespace"), name=table_info.get("table_name"), create_address=False).to_dict() - schedule_logger(self.job_id).info(f'get table meta success: {table_meta_dict}') - table_meta_dict["part_of_data"] = serialize_b64(table_meta_dict["part_of_data"], to_str=True) - table_meta_dict["schema"] = serialize_b64(table_meta_dict["schema"], to_str=True) - return table_meta_dict - - def get_output_data_table(self, output_data_infos, tracker_client=None): - """ - Get component output data table, will run in the task executor process - :param output_data_infos: - :return: - """ - output_tables_meta = {} - if output_data_infos: - for output_data_info in output_data_infos: - schedule_logger(self.job_id).info("get task {} {} output table {} {}".format(output_data_info.f_task_id, output_data_info.f_task_version, output_data_info.f_table_namespace, output_data_info.f_table_name)) - if not tracker_client: - data_table_meta = storage.StorageTableMeta(name=output_data_info.f_table_name, namespace=output_data_info.f_table_namespace) - else: - data_table_meta = tracker_client.get_table_meta(output_data_info.f_table_name, output_data_info.f_table_namespace) - - output_tables_meta[output_data_info.f_data_name] = data_table_meta - return output_tables_meta - - def save_output_cache(self, cache_data: typing.Dict[str, CTableABC], cache_meta: dict, cache_name, output_storage_engine, output_storage_address: dict, token=None): - output_namespace, output_name = default_output_info(task_id=self.task_id, task_version=self.task_version, output_type="cache") - cache = CacheManager.persistent(cache_name, cache_data, cache_meta, output_namespace, output_name, output_storage_engine, output_storage_address, token=token) - cache_key = self.tracking_output_cache(cache=cache, cache_name=cache_name) - return cache_key - - def tracking_output_cache(self, cache: DataCache, cache_name: str) -> str: - cache_key = CacheManager.record(cache=cache, - job_id=self.job_id, - role=self.role, - party_id=self.party_id, - component_name=self.component_name, - task_id=self.task_id, - task_version=self.task_version, - cache_name=cache_name) - schedule_logger(self.job_id).info(f"tracking {self.task_id} {self.task_version} output cache, cache key is {cache_key}") - return cache_key - - def get_output_cache(self, cache_key=None, cache_name=None): - caches = self.query_output_cache(cache_key=cache_key, cache_name=cache_name) - if caches: - return CacheManager.load(cache=caches[0]) - else: - return None, None - - def query_output_cache(self, cache_key=None, cache_name=None) -> typing.List[DataCache]: - caches = CacheManager.query(job_id=self.job_id, role=self.role, party_id=self.party_id, component_name=self.component_name, cache_name=cache_name, cache_key=cache_key) - group = {} - # only the latest version of the task output is retrieved - for cache in caches: - group_key = f"{cache.task_id}-{cache.name}" - if group_key not in group: - group[group_key] = cache - elif cache.task_version > group[group_key].task_version: - group[group_key] = cache - return list(group.values()) - - def query_output_cache_record(self): - return CacheManager.query_record(job_id=self.job_id, role=self.role, party_id=self.party_id, component_name=self.component_name, - task_version=self.task_version) - - @DB.connection_context() - def insert_summary_into_db(self, summary_data: dict, need_serialize=True): - try: - summary_model = self.get_dynamic_db_model(ComponentSummary, self.job_id) - DB.create_tables([summary_model]) - summary_obj = summary_model.get_or_none( - summary_model.f_job_id == self.job_id, - summary_model.f_component_name == self.component_name, - summary_model.f_role == self.role, - summary_model.f_party_id == self.party_id, - summary_model.f_task_id == self.task_id, - summary_model.f_task_version == self.task_version - ) - if summary_obj: - summary_obj.f_summary = serialize_b64(summary_data, to_str=True) if need_serialize else summary_data - summary_obj.f_update_time = current_timestamp() - summary_obj.save() - else: - self.get_dynamic_db_model(ComponentSummary, self.job_id).create( - f_job_id=self.job_id, - f_component_name=self.component_name, - f_role=self.role, - f_party_id=self.party_id, - f_task_id=self.task_id, - f_task_version=self.task_version, - f_summary=serialize_b64(summary_data, to_str=True), - f_create_time=current_timestamp() - ) - except Exception as e: - schedule_logger(self.job_id).exception("An exception where querying summary job id: {} " - "component name: {} to database:\n{}".format( - self.job_id, self.component_name, e) - ) - - @DB.connection_context() - def read_summary_from_db(self, need_deserialize=True): - cpn_summary = "" - try: - summary_model = self.get_dynamic_db_model(ComponentSummary, self.job_id) - summary = summary_model.get_or_none( - summary_model.f_job_id == self.job_id, - summary_model.f_component_name == self.component_name, - summary_model.f_role == self.role, - summary_model.f_party_id == self.party_id - ) - if summary: - cpn_summary = deserialize_b64(summary.f_summary) if need_deserialize else summary.f_summary - except Exception as e: - schedule_logger(self.job_id).exception(e) - return cpn_summary - - @DB.connection_context() - def reload_summary(self, source_tracker): - cpn_summary = source_tracker.read_summary_from_db(need_deserialize=False) - if cpn_summary: - self.insert_summary_into_db(cpn_summary, need_serialize=False) - - def log_output_data_info(self, data_name: str, table_namespace: str, table_name: str): - self.insert_output_data_info_into_db(data_name=data_name, table_namespace=table_namespace, table_name=table_name) - - @DB.connection_context() - def insert_output_data_info_into_db(self, data_name: str, table_namespace: str, table_name: str): - try: - tracking_output_data_info = self.get_dynamic_db_model(TrackingOutputDataInfo, self.job_id)() - tracking_output_data_info.f_job_id = self.job_id - tracking_output_data_info.f_component_name = self.component_name - tracking_output_data_info.f_task_id = self.task_id - tracking_output_data_info.f_task_version = self.task_version - tracking_output_data_info.f_data_name = data_name - tracking_output_data_info.f_role = self.role - tracking_output_data_info.f_party_id = self.party_id - tracking_output_data_info.f_table_namespace = table_namespace - tracking_output_data_info.f_table_name = table_name - tracking_output_data_info.f_create_time = current_timestamp() - - bulk_insert_into_db( - self.get_dynamic_db_model(TrackingOutputDataInfo, self.job_id), - (tracking_output_data_info.to_dict(), ), - ) - except Exception as e: - schedule_logger(self.job_id).exception("An exception where inserted output data info {} {} {} to database:\n{}".format( - data_name, - table_namespace, - table_name, - e - )) - - def save_as_table(self, computing_table, name, namespace): - if self.job_parameters.storage_engine == StorageEngine.LINKIS_HIVE: - return - self.save_output_data(computing_table=computing_table, - output_storage_engine=self.job_parameters.storage_engine, - output_storage_address=self.job_parameters.engines_address.get(EngineType.STORAGE, {}), - output_table_namespace=namespace, output_table_name=name) - - @DB.connection_context() - def clean_metrics(self): - return self.metric_manager.clean_metrics() - - @DB.connection_context() - def get_metric_list(self, job_level: bool = False): - return self.metric_manager.get_metric_list(job_level=job_level) - - @DB.connection_context() - def reload_metric(self, source_tracker): - return self.metric_manager.reload_metric(source_tracker.metric_manager) - - def get_output_data_info(self, data_name=None): - return self.read_output_data_info_from_db(data_name=data_name) - - def read_output_data_info_from_db(self, data_name=None): - filter_dict = {} - filter_dict["job_id"] = self.job_id - filter_dict["component_name"] = self.component_name - filter_dict["role"] = self.role - filter_dict["party_id"] = self.party_id - if data_name: - filter_dict["data_name"] = data_name - return self.query_output_data_infos(**filter_dict) - - @classmethod - @DB.connection_context() - def query_output_data_infos(cls, **kwargs) -> typing.List[TrackingOutputDataInfo]: - try: - tracking_output_data_info_model = cls.get_dynamic_db_model(TrackingOutputDataInfo, kwargs.get("job_id")) - filters = [] - for f_n, f_v in kwargs.items(): - attr_name = 'f_%s' % f_n - if hasattr(tracking_output_data_info_model, attr_name): - filters.append(operator.attrgetter('f_%s' % f_n)(tracking_output_data_info_model) == f_v) - if filters: - output_data_infos_tmp = tracking_output_data_info_model.select().where(*filters) - else: - output_data_infos_tmp = tracking_output_data_info_model.select() - output_data_infos_group = {} - # only the latest version of the task output data is retrieved - for output_data_info in output_data_infos_tmp: - group_key = cls.get_output_data_group_key(output_data_info.f_task_id, output_data_info.f_data_name) - if group_key not in output_data_infos_group: - output_data_infos_group[group_key] = output_data_info - elif output_data_info.f_task_version > output_data_infos_group[group_key].f_task_version: - output_data_infos_group[group_key] = output_data_info - return list(output_data_infos_group.values()) - except Exception as e: - return [] - - @classmethod - def get_output_data_group_key(cls, task_id, data_name): - return task_id + data_name - - def clean_task(self): - schedule_logger(self.job_id).info( - 'clean task {} {} on {} {}'.format(self.task_id, self.task_version, self.role, self.party_id)) - session_id = job_utils.generate_session_id(self.task_id, self.task_version, self.role, self.party_id) - sess = session.Session(session_id=session_id, options={"logger": schedule_logger(self.job_id)}) - sess.destroy_all_sessions() - return True - - @classmethod - def get_dynamic_db_model(cls, base, job_id): - return type(base.model(table_index=cls.get_dynamic_tracking_table_index(job_id=job_id))) - - @classmethod - def get_dynamic_tracking_table_index(cls, job_id): - return job_id[:8] diff --git a/python/fate_flow/pipelined_model/__init__.py b/python/fate_flow/pipelined_model/__init__.py deleted file mode 100644 index b68d67ac1..000000000 --- a/python/fate_flow/pipelined_model/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -class Pipelined: - - def __init__(self, *, role=None, party_id=None, model_id=None, party_model_id=None, model_version): - if party_model_id is None: - self.role = role - self.party_id = party_id - self.model_id = model_id - self.party_model_id = f'{role}#{party_id}#{model_id}' - else: - self.role, self.party_id, self.model_id = party_model_id.split('#', 2) - self.party_model_id = party_model_id - - self.model_version = model_version diff --git a/python/fate_flow/pipelined_model/deploy_model.py b/python/fate_flow/pipelined_model/deploy_model.py deleted file mode 100644 index 287437fe1..000000000 --- a/python/fate_flow/pipelined_model/deploy_model.py +++ /dev/null @@ -1,227 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import shutil - -from fate_arch.common.base_utils import json_dumps, json_loads - -from fate_flow.db.db_models import PipelineComponentMeta -from fate_flow.model.checkpoint import CheckpointManager -from fate_flow.model.sync_model import SyncComponent, SyncModel -from fate_flow.operation.job_saver import JobSaver -from fate_flow.pipelined_model.pipelined_model import PipelinedModel -from fate_flow.settings import ENABLE_MODEL_STORE, stat_logger -from fate_flow.utils.base_utils import compare_version -from fate_flow.utils.config_adapter import JobRuntimeConfigAdapter -from fate_flow.utils.model_utils import ( - check_before_deploy, gather_model_info_data, - gen_party_model_id, save_model_info, -) -from fate_flow.utils.schedule_utils import get_dsl_parser_by_version - - -def deploy(config_data): - model_id = config_data['model_id'] - model_version = config_data['model_version'] - local_role = config_data['local']['role'] - local_party_id = config_data['local']['party_id'] - child_model_version = config_data['child_model_version'] - components_checkpoint = config_data.get('components_checkpoint', {}) - warning_msg = "" - - try: - if ENABLE_MODEL_STORE: - sync_model = SyncModel( - role=local_role, party_id=local_party_id, - model_id=model_id, model_version=model_version, - ) - if sync_model.remote_exists(): - sync_model.download(True) - - party_model_id = gen_party_model_id( - model_id=model_id, - role=local_role, - party_id=local_party_id, - ) - source_model = PipelinedModel(party_model_id, model_version) - deploy_model = PipelinedModel(party_model_id, child_model_version) - - if not source_model.exists(): - raise FileNotFoundError(f'Can not found {model_id} {model_version} model local cache.') - - if not check_before_deploy(source_model): - raise Exception('Child model could not be deployed.') - - pipeline_model = source_model.read_pipeline_model() - pipeline_model.model_version = child_model_version - - train_runtime_conf = json_loads(pipeline_model.train_runtime_conf) - dsl_version = int(train_runtime_conf.get('dsl_version', 1)) - parser = get_dsl_parser_by_version(dsl_version) - - inference_dsl = config_data.get('predict_dsl', config_data.get('dsl')) - - if inference_dsl is not None: - if dsl_version == 1: - raise KeyError("'predict_dsl' is not supported in DSL v1") - - if 'cpn_list' in config_data: - raise KeyError("'cpn_list' should not be set when 'predict_dsl' is set") - - if not isinstance(inference_dsl, dict): - inference_dsl = json_loads(inference_dsl) - else: - if dsl_version == 1: - if 'cpn_list' in config_data: - raise KeyError("'cpn_list' is not supported in DSL v1") - - inference_dsl, warning_msg = parser.convert_dsl_v1_to_v2( - json_loads(pipeline_model.inference_dsl), - ) - else: - train_dsl = json_loads(pipeline_model.train_dsl) - inference_dsl = parser.deploy_component( - config_data.get( - 'cpn_list', - list(train_dsl.get('components', {}).keys()), - ), - train_dsl, - ) - - cpn_list = list(inference_dsl.get('components', {}).keys()) - - if dsl_version == 1: - from fate_flow.db.component_registry import ComponentRegistry - - job_providers = parser.get_job_providers( - dsl=inference_dsl, - provider_detail=ComponentRegistry.REGISTRY, - ) - train_runtime_conf = parser.convert_conf_v1_to_v2( - train_runtime_conf, - { - cpn: parser.parse_component_role_parameters( - component=cpn, - dsl=inference_dsl, - runtime_conf=train_runtime_conf, - provider_detail=ComponentRegistry.REGISTRY, - provider_name=job_providers[cpn]['provider']['name'], - provider_version=job_providers[cpn]['provider']['version'], - ) for cpn in cpn_list - } - ) - - parser = get_dsl_parser_by_version() - parser.verify_dsl(inference_dsl, 'predict') - inference_dsl = JobSaver.fill_job_inference_dsl( - job_id=model_version, role=local_role, party_id=local_party_id, - dsl_parser=parser, origin_inference_dsl=inference_dsl, - ) - # migrate model miss CodePath - module_object_dict = get_module_object_dict(json_loads(pipeline_model.inference_dsl)) - pipeline_model.inference_dsl = json_dumps(parser.get_predict_dsl(inference_dsl, module_object_dict), byte=True) - - train_runtime_conf = JobRuntimeConfigAdapter( - train_runtime_conf, - ).update_model_id_version( - model_version=child_model_version, - ) - pipeline_model.train_runtime_conf = json_dumps(train_runtime_conf, byte=True) - - if compare_version(pipeline_model.fate_version, '1.5.0') == 'gt': - runtime_conf_on_party = json_loads(pipeline_model.runtime_conf_on_party) - runtime_conf_on_party['job_parameters']['model_version'] = child_model_version - pipeline_model.runtime_conf_on_party = json_dumps(runtime_conf_on_party, byte=True) - - pipeline_model.parent = False - pipeline_model.parent_info = json_dumps({ - 'parent_model_id': model_id, - 'parent_model_version': model_version, - }, byte=True) - - query_args = ( - PipelineComponentMeta.f_component_name.in_(cpn_list), - ) - query = source_model.pipelined_component.get_define_meta_from_db(*query_args) - - for row in query: - shutil.copytree( - source_model.pipelined_component.variables_data_path / row.f_component_name, - deploy_model.pipelined_component.variables_data_path / row.f_component_name, - ) - - source_model.pipelined_component.replicate_define_meta({ - 'f_model_version': child_model_version, - 'f_archive_sha256': None, - 'f_archive_from_ip': None, - }, query_args) - - if ENABLE_MODEL_STORE: - for row in query: - sync_component = SyncComponent( - role=local_role, party_id=local_party_id, - model_id=model_id, model_version=child_model_version, - component_name=row.f_component_name, - ) - sync_component.copy(model_version, row.f_archive_sha256) - - deploy_model.save_pipeline_model(pipeline_model) - - for row in query: - step_index = components_checkpoint.get(row.f_component_name, {}).get('step_index') - step_name = components_checkpoint.get(row.f_component_name, {}).get('step_name') - if step_index is not None: - step_index = int(step_index) - step_name = None - elif step_name is None: - continue - - checkpoint_manager = CheckpointManager( - role=local_role, party_id=local_party_id, - model_id=model_id, model_version=model_version, - component_name=row.f_component_name, - ) - checkpoint_manager.load_checkpoints_from_disk() - if checkpoint_manager.latest_checkpoint is not None: - checkpoint_manager.deploy( - child_model_version, - row.f_model_alias, - step_index, - step_name, - ) - - deploy_model_info = gather_model_info_data(deploy_model) - save_model_info(deploy_model_info) - except Exception as e: - stat_logger.exception(e) - - return 100, ( - f'deploy model of role {local_role} {local_party_id} failed, ' - f'details: {repr(e)}' - ) - else: - return 0, ( - f'deploy model of role {local_role} {local_party_id} success' - + ('' if not warning_msg else f', warning: {warning_msg}') - ) - - -def get_module_object_dict(inference_dsl): - module_object_dict = {} - for _, components in inference_dsl.items(): - for name, module in components.items(): - module_object_dict[name] = module.get("CodePath") - return module_object_dict - diff --git a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/__init__.py b/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/__init__.py deleted file mode 100644 index e6bb47da2..000000000 --- a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/__init__.py +++ /dev/null @@ -1,56 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -def get_kfserving_deployer(party_model_id, - model_version, - model_object, - framework_name, - service_id, - protocol_version="v1", - **kwargs): - """ - Returns a deployer for KFServing InferenceService - - Refer to KFServingDeployer and its sub-classes - for supported kwargs. - :param party_model_id: the model id with party info used to identify the model - :param model_version: the model version - :param model_object: the converted model object - :param framework_name: the framework of the model_object - :param service_id: name of the serving service, will be used in KFServing as the service name - :param protocol_version: the protocol version, currently only scikit-learn model supports v2 - :param kwargs: keyword argument to initialize the deployer object - :return: an instance of the subclass of the base KFServingDeployer - """ - if framework_name in ['sklearn', 'scikit-learn']: - from .sklearn import SKLearnV1KFDeployer, SKLearnV2KFDeployer - if protocol_version == "v2": - cls = SKLearnV2KFDeployer - else: - cls = SKLearnV1KFDeployer - elif framework_name in ['pytorch', 'torch']: - from .pytorch import TorchServeKFDeployer - cls = TorchServeKFDeployer - elif framework_name in ['tf_keras', 'tensorflow', 'tf']: - from .tensorflow import TFServingKFDeployer - cls = TFServingKFDeployer - elif framework_name in ['lightgbm']: - from .lightgbm import LightGBMKFDeployer - cls = LightGBMKFDeployer - else: - raise ValueError("unknown converted model framework: {}".format(framework_name)) - return cls(party_model_id, model_version, model_object, service_id, **kwargs) diff --git a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/_dummy_base_handler.py b/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/_dummy_base_handler.py deleted file mode 100644 index ce14b7ec7..000000000 --- a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/_dummy_base_handler.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This file won't be loaded in fate_flow but will be packaged into the torch model archive -from ts.torch_handler.base_handler import BaseHandler - - -class DummyHandler(BaseHandler): - pass diff --git a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/base.py b/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/base.py deleted file mode 100644 index fc6b55363..000000000 --- a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/base.py +++ /dev/null @@ -1,225 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import io -import uuid -import kfserving -from kfserving.api import creds_utils -from kubernetes import client - -from fate_flow.settings import stat_logger -from .model_storage import get_model_storage, ModelStorageType -from .model_storage.minio import MinIOModelStorage - -MINIO_K8S_SECRET_NAME = "fate-homo-serving-minio-secret" - -STORAGE_URI_KEY = "storage_uri" - -ANNOTATION_PREFIX = "fate.fedai.org/" -ANNOTATION_SERVICE_UUID = ANNOTATION_PREFIX + "uuid" -ANNOTATION_FATE_MODEL_ID = ANNOTATION_PREFIX + "model_id" -ANNOTATION_FATE_MODEL_VERSION = ANNOTATION_PREFIX + "model_version" - - -class KFServingDeployer(object): - """Class representing a KFServing service deployer - """ - - def __init__(self, - party_model_id, - model_version, - model_object, - service_id, - namespace=None, - config_file_content=None, - replace=False, - skip_create_storage_secret=False, - model_storage_type=ModelStorageType.MINIO, - model_storage_parameters=None): - """ - :param party_model_id: the model id with party info used to identify the model - :param model_version: the model version - :param model_object: the converted model object - :param service_id: name of the service - :param config_file_content: the content of a config file that will be used to connect to the - kubernetes cluster. - :param namespace: the kubernetes namespace this service belongs to. - :param replace: whether to replace the running service, defaults to False. - :param skip_create_storage_secret: whether or not to skip setting up MinIO credentials for - KFServing storage-initializer container, defaults to False. - :param model_storage_type: type of the underlying model storage - defaults to ModelStorageType.MINIO. - :param model_storage_parameters: a dict containing extra arguments to initialize the - model storage instance, defaults to {}. - see the doc of model storage classes for the available - parameters. - """ - self.party_model_id = party_model_id - self.model_version = model_version - self.model_object = model_object - self.service_id = service_id - if model_storage_parameters is None: - model_storage_parameters = {} - self.model_storage = get_model_storage(storage_type=model_storage_type, - sub_path=party_model_id + "/" + model_version + "/" + service_id, - **model_storage_parameters) - self.storage_uri = None - self.isvc = None - # this should also set up kubernetes.client config - config_file = None - if config_file_content: - config_file = io.StringIO(config_file_content) - self.kfserving_client = kfserving.KFServingClient(config_file) - self.namespace = namespace if namespace else kfserving.utils.get_default_target_namespace() - self.replace = replace - self.skip_create_storage_secret = skip_create_storage_secret - stat_logger.debug("Initialized KFServingDeployer with client config: {}".format(config_file)) - - def prepare_model(self): - """ - Prepare the model to be used by KFServing system. - - Calls into each deployer implementation to serialize the model object - and uploads the related files to the target model storage. - - :return: the uri to fetch the uploaed/prepared model - """ - if not self.storage_uri: - self.storage_uri = self.model_storage.save(self._do_prepare_model()) - stat_logger.info("Prepared model with uri: {}".format(self.storage_uri)) - return self.storage_uri - - def _do_prepare_model(self): - raise NotImplementedError("_do_prepare_storage method not implemented") - - def deploy(self): - """ - Deploy a KFServing InferenceService from a model object - - :return: the InferenceService object as a dict - """ - if self.status() and not self.replace: - raise RuntimeError("serving service {} already exists".format(self.service_id)) - - if self.isvc is None: - stat_logger.info("Preparing model storage and InferenceService spec...") - self.prepare_model() - self.prepare_isvc() - if self.isvc.metadata.annotations is None: - self.isvc.metadata.annotations = {} - # add a different annotation to force replace - self.isvc.metadata.annotations[ANNOTATION_SERVICE_UUID] = str(uuid.uuid4()) - self.isvc.metadata.annotations[ANNOTATION_FATE_MODEL_ID] = self.party_model_id - self.isvc.metadata.annotations[ANNOTATION_FATE_MODEL_VERSION] = self.model_version - - if isinstance(self.model_storage, MinIOModelStorage) and not self.skip_create_storage_secret: - self.prepare_sa_secret() - - if self.status() is None: - stat_logger.info("Creating InferenceService {}...".format(self.service_id)) - created_isvc = self.kfserving_client.create(self.isvc, namespace=self.namespace) - else: - stat_logger.info("Replacing InferenceService {}...".format(self.service_id)) - self.isvc.metadata.resource_version = None - created_isvc = self.kfserving_client.replace(self.service_id, self.isvc, - namespace=self.namespace) - return created_isvc - - def prepare_isvc(self): - """ - Generate an InferenceService spec to be applied into KFServing - - :return: the spec object - """ - if self.isvc is None: - self.isvc = kfserving.V1beta1InferenceService( - api_version=kfserving.constants.KFSERVING_V1BETA1, - kind=kfserving.constants.KFSERVING_KIND, - metadata=client.V1ObjectMeta(name=self.service_id), - spec=kfserving.V1beta1InferenceServiceSpec( - predictor=kfserving.V1beta1PredictorSpec())) - self._do_prepare_predictor() - if self.namespace: - self.isvc.metadata.namespace = self.namespace - stat_logger.info("InferenceService spec ready") - stat_logger.debug(self.isvc) - return self.isvc - - def _do_prepare_predictor(self): - raise NotImplementedError("_do_prepare_predictor method not implemented") - - def destroy(self): - """ - Delete the InferenceService - """ - if self.status() is not None: - self.kfserving_client.delete(self.service_id, namespace=self.namespace) - stat_logger.info("InferenceService {} is deleted".format(self.service_id)) - - def status(self): - try: - return self.kfserving_client.get(self.service_id, namespace=self.namespace) - except RuntimeError as e: - if "Reason: Not Found" in str(e): - return None - - def wait(self, timeout=120): - """Wait until the service becomes ready - - Internally calls KFServing API to retrieve the status - - :param timeout: seconds to wait - :return: the InferenceService dict - """ - return self.kfserving_client.get(self.service_id, - namespace=self.namespace, - watch=True, - timeout_seconds=timeout) - - def prepare_sa_secret(self): - """ - Prepare the secret to be used by the service account for the KFServing service. - - KFServing needs a service account to find the credential to download files - from MINIO/S3 storage. It must contain a secret resource with the credential - embedded. We can prepare one use kubernetes API here. - """ - secrets = client.CoreV1Api().list_namespaced_secret(self.namespace) - secret_names = [secret.metadata.name for secret in secrets.items] - annotations = { - "serving.kubeflow.org/s3-endpoint": self.model_storage.endpoint, - "serving.kubeflow.org/s3-usehttps": "1" if self.model_storage.secure else "0" - } - secret = client.V1Secret(metadata=client.V1ObjectMeta(name=MINIO_K8S_SECRET_NAME, - annotations=annotations), - type="Opaque", - string_data={ - 'AWS_ACCESS_KEY_ID': self.model_storage.access_key, - 'AWS_SECRET_ACCESS_KEY': self.model_storage.secret_key - }) - if MINIO_K8S_SECRET_NAME not in secret_names: - client.CoreV1Api().create_namespaced_secret(self.namespace, secret) - else: - client.CoreV1Api().patch_namespaced_secret(MINIO_K8S_SECRET_NAME, self.namespace, secret) - - sa_name = self.isvc.spec.predictor.service_account_name \ - if (self.isvc and - isinstance(self.isvc, kfserving.V1beta1InferenceService) and - self.isvc.spec.predictor.service_account_name) \ - else "default" - creds_utils.set_service_account(self.namespace, - sa_name, - MINIO_K8S_SECRET_NAME) diff --git a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/lightgbm.py b/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/lightgbm.py deleted file mode 100644 index b69211a64..000000000 --- a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/lightgbm.py +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import os -import tempfile -from abc import ABC - -import kfserving - -from .base import KFServingDeployer - - -class LightGBMKFDeployer(KFServingDeployer, ABC): - def _do_prepare_model(self): - working_dir = tempfile.mkdtemp() - local_file = os.path.join(working_dir, "model.bst") - self.model_object.save_model(local_file) - return local_file - - def _do_prepare_predictor(self): - self.isvc.spec.predictor.lightgbm = kfserving.V1beta1LightGBMSpec(storage_uri=self.storage_uri) diff --git a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/model_storage/__init__.py b/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/model_storage/__init__.py deleted file mode 100644 index 9748e9068..000000000 --- a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/model_storage/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from enum import Enum - -from .minio import MinIOModelStorage - - -class ModelStorageType(Enum): - MINIO = 1 - - -def get_model_storage(storage_type=ModelStorageType.MINIO, **kwargs): - """ - get a model storage insance based on the specified type - - :param storage_type: type of the model storage, currently only MINIO is supported - :param kwargs: keyword arguments to initialize the model storage object - :return: an instance of a subclass of BaseModelStorage - """ - if isinstance(storage_type, str): - storage_type = ModelStorageType[storage_type.upper()] - if storage_type == ModelStorageType.MINIO: - return MinIOModelStorage(**kwargs) - else: - raise ValueError("unknown model storage type: {}".format(storage_type)) diff --git a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/model_storage/base.py b/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/model_storage/base.py deleted file mode 100644 index 2af0f3cb5..000000000 --- a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/model_storage/base.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -class BaseModelStorage(object): - def save(self, model_object, dest): - pass diff --git a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/model_storage/minio.py b/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/model_storage/minio.py deleted file mode 100644 index 64015e2e2..000000000 --- a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/model_storage/minio.py +++ /dev/null @@ -1,177 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -from urllib.parse import urlparse -from minio import Minio - -from fate_flow.settings import stat_logger -from .base import BaseModelStorage - - -ENV_MINIO_ENDPOINT_URL = "MINIO_ENDPOINT_URL" -ENV_MINIO_ACCESS_KEY_ID = "MINIO_ACCESS_KEY_ID" -ENV_MINIO_SECRET_ACCESS_KEY = "MINIO_SECRET_ACCESS_KEY" -ENV_MINIO_USE_HTTPS = "MINIO_USE_HTTPS" -ENV_MINIO_REGION = "MINIO_REGION" - - -class MinIOModelStorage(BaseModelStorage): - """Model storage to upload model object into MinIO - - If not specified by the caller, the following environment - variables will be used to connect to the server: - MINIO_ENDPOINT_URL, MINIO_ACCESS_KEY_ID, MINIO_SECRET_ACCESS_KEY - MINIO_USE_HTTPS, MINIO_REGION. - """ - - def __init__(self, - sub_path="", - bucket="fate-models", - endpoint=None, - access_key=None, - secret_key=None, - region=None, - secure=True): - """ - :param sub_path: sub path within the bucket, defaults to "" - :param bucket: bucket to store the model, defaults to "fate_models" - :param endpoint: server endpoint, defaults to None - :param access_key: access key, defaults to None - :param secret_key: secret key, defaults to None - :param region: region name, defaults to None - :param secure: flag to indicate whether tls should be used, defaults to True - """ - super(MinIOModelStorage, self).__init__() - self.bucket = bucket - # When parsing the uri, KFServing's storage initializer will incorrectly truncate it if - # it has pound key in it. Replace with another character. - self.sub_path = sub_path.replace("#", "^") - - if not endpoint: - url = urlparse(os.getenv(ENV_MINIO_ENDPOINT_URL, "http://minio:9000")) - secure = url.scheme == 'https' if url.scheme else bool(os.getenv(ENV_MINIO_USE_HTTPS, "true")) - endpoint = url.netloc - access_key = os.getenv(ENV_MINIO_ACCESS_KEY_ID) - secret_key = os.getenv(ENV_MINIO_SECRET_ACCESS_KEY) - region = os.getenv(ENV_MINIO_REGION) - - stat_logger.debug("Initialized MinIOModelStorage with endpoint: {}, " - "access_key: {}, region: {}, with TLS: {}" - .format(endpoint, access_key, region, secure)) - self.endpoint = endpoint - self.access_key = access_key - self.secret_key = secret_key - self.region = region - self.secure = secure - - self.client = Minio( - endpoint, - access_key=access_key, - secret_key=secret_key, - region=region, - secure=secure - ) - - def save(self, model_object, dest=""): - """ - Upload the model objects to the specified destination path - - If the dest parameter is empty, the target file/object name - will be inferred from the local files. - - :param model_object: pointer to the file(s) to be uploaded - :param dest: the destination file/object name - :return: - """ - stat_logger.debug("Upload model object: {} of type: {}" - .format(model_object, type(model_object))) - # If the model object is already a local file then we - # just upload it. - if isinstance(model_object, str): - if os.path.isfile(model_object): - return self.upload_file(model_object, dest) - elif os.path.isdir(model_object): - return self.upload_folder(model_object, dest) - else: - raise ValueError("expect an existing path, got: {}".format(model_object)) - elif isinstance(model_object, list): - for obj in model_object: - if 'dest' not in obj: - obj['dest'] = os.path.basename(obj['file']) - return self.upload_objects(model_object) - elif isinstance(model_object, dict) and "dest" in model_object \ - and "file" in model_object and isinstance(model_object['file'], str): - return self.upload_objects([model_object]) - else: - raise ValueError("unsupported object type {}".format(type(model_object))) - - def upload_folder(self, folder, dest=""): - files_to_upload = [] - for dir_, _, files in os.walk(folder): - for file_name in files: - rel_dir = os.path.relpath(dir_, folder).lstrip("./") - rel_dest = os.path.join(rel_dir, file_name) - if dest: - rel_dest = dest + "/" + rel_dest - file = os.path.join(dir_, file_name) - files_to_upload.append({"dest": rel_dest, "obj": file}) - return self.upload_objects(files_to_upload) - - def upload_file(self, file, dest=""): - if not dest: - dest = os.path.basename(file) - return self.upload_objects([{"dest": dest, "obj": file}]) - - def upload_readable_obj(self, dest, obj, length): - return self.upload_objects([{"dest": dest, - "obj": obj, - "length": length - }]) - - def upload_objects(self, objects): - """ - Perform the uploading. - - Each element in the objects list should be a dict that looks: - - { - "obj": - "dest": - "length": - } - - :param objects: a list of object to be uploaded. - :return: the complete "s3://" type uri for the sub-path to local all the uploaded objects - """ - client = self.client - bucket = self.bucket - found = client.bucket_exists(bucket) - if not found: - client.make_bucket(bucket) - - for obj in objects: - dest = self.sub_path + "/" + obj['dest'] - file = obj['obj'] - stat_logger.debug("Uploading {} to {}".format(file, dest)) - if hasattr(file, 'read'): - length = obj['length'] - client.put_object(bucket, dest, file, length) - else: - client.fput_object(bucket, dest, file) - model_path = f's3://{bucket}/{self.sub_path}' - stat_logger.info("Uploaded model objects into path: {}".format(model_path)) - return model_path diff --git a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/pytorch.py b/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/pytorch.py deleted file mode 100644 index 8a5a68478..000000000 --- a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/pytorch.py +++ /dev/null @@ -1,119 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import json -import kfserving -import os -import subprocess -import tempfile -import torch - -from .base import KFServingDeployer - -CONFIG_FILE_DIR = "config" -CONFIG_FILE_NAME = "config.properties" -CONFIG_FILE = CONFIG_FILE_DIR + "/" + CONFIG_FILE_NAME - -MODEL_STORE_DIR = "model-store" - -_ts_config = '''inference_address=http://0.0.0.0:8085 -management_address=http://0.0.0.0:8081 -metrics_address=http://0.0.0.0:8082 -enable_metrics_api=true -metrics_format=prometheus -number_of_netty_threads=4 -job_queue_size=10 -service_envelope=kfserving -model_store=/mnt/models/model-store -model_snapshot={}''' - - -def generate_files(working_dir, - model_name, - torch_model, - handler="", - min_workers=1, - max_workers=5, - batch_size=1, - max_batch_delay=5000, - response_timeout=120): - """generate mar file and config file - - The defaults are chosen from - https://github.com/kubeflow/kfserving/blob/v0.5.1/docs/samples/v1beta1/torchserve/config.properties - """ - torch_script = torch.jit.script(torch_model) - torch_script_file = os.path.join(working_dir, model_name + ".pt") - torch_script.save(torch_script_file) - if not handler: - handler = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_dummy_base_handler.py") - - # there is no official support of using the model-archiver package via python function calls - command = ["torch-model-archiver", - "--serialized-file", - torch_script_file, - "--model-name", - model_name, - "--export-path", - working_dir, - "--handler", - handler, - "-v", - "1.0", - "-f"] - ret = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - if ret.returncode != 0: - raise RuntimeError("error running torch-model-archiver: {}, command output: {}".format(ret.args, ret.stdout)) - mar_file = os.path.join(working_dir, model_name + ".mar") - - model_snapshot_dict = { - "name": "startup.cfg", - "modelCount": 1, - "models": { - model_name: { - "1.0": { - "defaultVersion": True, - "marName": os.path.basename(mar_file), - "minWorkers": min_workers, - "maxWorkers": max_workers, - "batchSize": batch_size, - "maxBatchDelay": max_batch_delay, - "responseTimeout": response_timeout - } - } - } - } - config = _ts_config.format(json.dumps(model_snapshot_dict)) - config_file = os.path.join(working_dir, CONFIG_FILE_NAME) - with open(config_file, "w") as f: - f.write(config) - - return mar_file, config_file - - -class TorchServeKFDeployer(KFServingDeployer): - def _do_prepare_model(self): - working_dir = tempfile.mkdtemp() - mar_file, config_file = generate_files(working_dir, - self.service_id, - self.model_object) - objects = [{"dest": CONFIG_FILE, "obj": config_file}, - {"dest": MODEL_STORE_DIR + "/" + os.path.basename(mar_file), "obj": mar_file}] - return objects - - def _do_prepare_predictor(self): - self.isvc.spec.predictor.pytorch = kfserving.V1beta1TorchServeSpec(storage_uri=self.storage_uri) diff --git a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/sklearn.py b/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/sklearn.py deleted file mode 100644 index b02caf11c..000000000 --- a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/sklearn.py +++ /dev/null @@ -1,50 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import os -import tempfile -from abc import ABC - -import joblib -import kfserving - -from .base import KFServingDeployer - - -class SKLearnKFDeployer(KFServingDeployer, ABC): - def _do_prepare_model(self): - working_dir = tempfile.mkdtemp() - local_file = os.path.join(working_dir, "model.joblib") - joblib.dump(self.model_object, local_file) - return local_file - - -class SKLearnV1KFDeployer(SKLearnKFDeployer): - def _do_prepare_predictor(self): - # Use our own sklearnserver image that has scikit-learn==0.24.2 because KFServing's default one - # uses scikit-learn==0.20.3 that cannot de-serialize models of higher versions. - self.isvc.spec.predictor.sklearn = kfserving.V1beta1SKLearnSpec( - image="federatedai/sklearnserver:v0.6.1-0.24.2", - protocol_version='v1', - storage_uri=self.storage_uri) - - -class SKLearnV2KFDeployer(SKLearnKFDeployer): - def _do_prepare_predictor(self): - self.isvc.spec.predictor.sklearn = kfserving.V1beta1SKLearnSpec( - protocol_version='v2', - storage_uri=self.storage_uri) diff --git a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/tensorflow.py b/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/tensorflow.py deleted file mode 100644 index 2679be099..000000000 --- a/python/fate_flow/pipelined_model/homo_model_deployer/kfserving/tensorflow.py +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import kfserving -import os -import tempfile -import tensorflow - -from .base import KFServingDeployer - - -class TFServingKFDeployer(KFServingDeployer): - def _do_prepare_model(self): - working_dir = tempfile.mkdtemp() - # TFServing expects an "version string" like 0001 in the model base path - local_folder = os.path.join(working_dir, '0001') - tensorflow.saved_model.save(self.model_object, local_folder) - return working_dir - - def _do_prepare_predictor(self): - self.isvc.spec.predictor.tensorflow = kfserving.V1beta1TFServingSpec(storage_uri=self.storage_uri) diff --git a/python/fate_flow/pipelined_model/homo_model_deployer/model_deploy.py b/python/fate_flow/pipelined_model/homo_model_deployer/model_deploy.py deleted file mode 100644 index 732627c7a..000000000 --- a/python/fate_flow/pipelined_model/homo_model_deployer/model_deploy.py +++ /dev/null @@ -1,51 +0,0 @@ -# -# Copyright 2021 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -from .kfserving import get_kfserving_deployer - - -def model_deploy(party_model_id, - model_version, - model_object, - framework_name, - service_id, - deployment_type, - deployment_parameters): - """ - Deploy a horizontally-trained model to a target serving system. - - Currently only KFServing is supported - - :param party_model_id: model id with party info to identify the model - :param model_version: model version - :param model_object: the converted model object - :param framework_name: the ML framework of the converted model - :param service_id: service name identifier - :param deployment_type: currently only "kfserving" is supported - :param deployment_parameters: parameters specific to the serving system - :return: the deployed service representation, defined by the serving system. - """ - if deployment_type == "kfserving": - deployer = get_kfserving_deployer(party_model_id, - model_version, - model_object, - framework_name, - service_id, - **deployment_parameters) - else: - raise ValueError("unknown deployment_type type: {}".format(deployment_type)) - return deployer.deploy() diff --git a/python/fate_flow/pipelined_model/migrate_model.py b/python/fate_flow/pipelined_model/migrate_model.py deleted file mode 100644 index e0638a59a..000000000 --- a/python/fate_flow/pipelined_model/migrate_model.py +++ /dev/null @@ -1,174 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_arch.common.base_utils import json_dumps, json_loads - -from fate_flow.db.db_models import DB, MachineLearningModelInfo as MLModel, PipelineComponentMeta -from fate_flow.model.sync_model import SyncModel -from fate_flow.pipelined_model import pipelined_model -from fate_flow.scheduler.cluster_scheduler import ClusterScheduler -from fate_flow.settings import ENABLE_MODEL_STORE, stat_logger -from fate_flow.utils.base_utils import compare_version -from fate_flow.utils.config_adapter import JobRuntimeConfigAdapter -from fate_flow.utils.job_utils import PIPELINE_COMPONENT_NAME -from fate_flow.utils.model_utils import ( - gather_model_info_data, gen_model_id, - gen_party_model_id, save_model_info, -) - - -def compare_roles(request_conf_roles: dict, run_time_conf_roles: dict): - if request_conf_roles.keys() == run_time_conf_roles.keys(): - verify_format = True - verify_equality = True - for key in request_conf_roles.keys(): - verify_format = ( - verify_format and - len(request_conf_roles[key]) == len(run_time_conf_roles[key]) and - isinstance(request_conf_roles[key], list) - ) - request_conf_roles_set = set(str(item) for item in request_conf_roles[key]) - run_time_conf_roles_set = set(str(item) for item in run_time_conf_roles[key]) - verify_equality = verify_equality and (request_conf_roles_set == run_time_conf_roles_set) - if not verify_format: - raise Exception("The structure of roles data of local configuration is different from " - "model runtime configuration's. Migration aborting.") - else: - return verify_equality - raise Exception("The structure of roles data of local configuration is different from " - "model runtime configuration's. Migration aborting.") - - -def migration(config_data: dict): - model_id = config_data['model_id'] - model_version = config_data['model_version'] - local_role = config_data['local']['role'] - local_party_id = config_data['local']['party_id'] - new_party_id = config_data["local"]["migrate_party_id"] - new_model_id = gen_model_id(config_data["migrate_role"]) - unify_model_version = config_data['unify_model_version'] - - try: - if ENABLE_MODEL_STORE: - sync_model = SyncModel( - role=local_role, party_id=local_party_id, - model_id=model_id, model_version=model_version, - ) - if sync_model.remote_exists(): - sync_model.download(True) - - party_model_id = gen_party_model_id( - model_id=model_id, - role=local_role, - party_id=local_party_id, - ) - source_model = pipelined_model.PipelinedModel(party_model_id, model_version) - if not source_model.exists(): - raise FileNotFoundError(f"Can not found {model_id} {model_version} model local cache.") - - with DB.connection_context(): - if MLModel.get_or_none( - MLModel.f_role == local_role, - MLModel.f_party_id == new_party_id, - MLModel.f_model_id == new_model_id, - MLModel.f_model_version == unify_model_version, - ): - raise FileExistsError( - f"Unify model version {unify_model_version} has been occupied in database. " - "Please choose another unify model version and try again." - ) - - migrate_tool = source_model.get_model_migrate_tool() - migrate_model = pipelined_model.PipelinedModel( - gen_party_model_id( - model_id=new_model_id, - role=local_role, - party_id=new_party_id, - ), - unify_model_version, - ) - - query = source_model.pipelined_component.get_define_meta_from_db( - PipelineComponentMeta.f_component_name != PIPELINE_COMPONENT_NAME, - ) - for row in query: - buffer_obj = source_model.read_component_model(row.f_component_name, row.f_model_alias) - - modified_buffer = migrate_tool.model_migration( - model_contents=buffer_obj, - module_name=row.f_component_module_name, - old_guest_list=config_data['role']['guest'], - new_guest_list=config_data['migrate_role']['guest'], - old_host_list=config_data['role']['host'], - new_host_list=config_data['migrate_role']['host'], - old_arbiter_list=config_data.get('role', {}).get('arbiter', None), - new_arbiter_list=config_data.get('migrate_role', {}).get('arbiter', None), - ) - - migrate_model.save_component_model( - row.f_component_name, row.f_component_module_name, - row.f_model_alias, modified_buffer, row.f_run_parameters, - ) - - pipeline_model = source_model.read_pipeline_model() - - pipeline_model.model_id = new_model_id - pipeline_model.model_version = unify_model_version - pipeline_model.roles = json_dumps(config_data['migrate_role'], byte=True) - - train_runtime_conf = json_loads(pipeline_model.train_runtime_conf) - train_runtime_conf['role'] = config_data['migrate_role'] - train_runtime_conf['initiator'] = config_data['migrate_initiator'] - train_runtime_conf = JobRuntimeConfigAdapter( - train_runtime_conf, - ).update_model_id_version( - model_id=new_model_id, - model_version=unify_model_version, - ) - pipeline_model.train_runtime_conf = json_dumps(train_runtime_conf, byte=True) - - if compare_version(pipeline_model.fate_version, '1.5.0') == 'gt': - pipeline_model.initiator_role = config_data["migrate_initiator"]['role'] - pipeline_model.initiator_party_id = config_data["migrate_initiator"]['party_id'] - - runtime_conf_on_party = json_loads(pipeline_model.runtime_conf_on_party) - runtime_conf_on_party['role'] = config_data['migrate_role'] - runtime_conf_on_party['initiator'] = config_data['migrate_initiator'] - runtime_conf_on_party['job_parameters']['model_id'] = new_model_id - runtime_conf_on_party['job_parameters']['model_version'] = unify_model_version - pipeline_model.runtime_conf_on_party = json_dumps(runtime_conf_on_party, byte=True) - - migrate_model.save_pipeline_model(pipeline_model) - - migrate_model_info = gather_model_info_data(migrate_model) - save_model_info(migrate_model_info) - - ClusterScheduler.cluster_command('/model/archive/packaging', { - 'party_model_id': migrate_model.party_model_id, - 'model_version': migrate_model.model_version, - }) - - return (0, ( - "Migrating model successfully. The configuration of model has been modified automatically. " - f"New model id is: {migrate_model._model_id}, model version is: {migrate_model.model_version}. " - f"Model files can be found at '{migrate_model.archive_model_file_path}'." - ), { - "model_id": migrate_model.party_model_id, - "model_version": migrate_model.model_version, - "path": migrate_model.archive_model_file_path, - }) - except Exception as e: - stat_logger.exception(e) - return 100, str(e), {} diff --git a/python/fate_flow/pipelined_model/pipelined_component.py b/python/fate_flow/pipelined_model/pipelined_component.py deleted file mode 100644 index 9b70b0a7d..000000000 --- a/python/fate_flow/pipelined_model/pipelined_component.py +++ /dev/null @@ -1,257 +0,0 @@ -# -# Copyright 2022 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import hashlib -import os -from pathlib import Path -from zipfile import ZipFile - -from ruamel import yaml - -from fate_arch.common.base_utils import json_dumps, json_loads - -from fate_flow.db.db_models import DB, PipelineComponentMeta -from fate_flow.db.db_utils import bulk_insert_into_db -from fate_flow.model import Locker, local_cache_required, lock -from fate_flow.pipelined_model import Pipelined -from fate_flow.settings import TEMP_DIRECTORY -from fate_flow.utils.base_utils import get_fate_flow_directory - - -class PipelinedComponent(Pipelined, Locker): - - def __init__(self, **kwargs): - Pipelined.__init__(self, **kwargs) - - self.model_path = Path(get_fate_flow_directory('model_local_cache'), self.party_model_id, self.model_version) - self.define_meta_path = self.model_path / 'define' / 'define_meta.yaml' - self.variables_data_path = self.model_path / 'variables' / 'data' - self.run_parameters_path = self.model_path / 'run_parameters' - self.checkpoint_path = self.model_path / 'checkpoint' - - self.query_args = ( - PipelineComponentMeta.f_model_id == self.model_id, - PipelineComponentMeta.f_model_version == self.model_version, - PipelineComponentMeta.f_role == self.role, - PipelineComponentMeta.f_party_id == self.party_id, - ) - - Locker.__init__(self, self.model_path) - - def exists(self, component_name=None, model_alias=None): - if component_name is None: - return self.model_path.is_dir() and set(os.listdir(self.model_path)) - {'.lock'} - - query = self.get_define_meta_from_db(PipelineComponentMeta.f_component_name == component_name) - - if query: - query = query[0] - - if model_alias is None: - model_alias = query.f_model_alias - - model_proto_index = query.f_model_proto_index - else: - query = self.get_define_meta_from_file() - - try: - query = query['model_proto'][component_name] - except KeyError: - return False - - if model_alias is None: - if len(query) != 1: - return False - - model_alias = next(iter(query.keys())) - - try: - model_proto_index = query[model_alias] - except KeyError: - return False - - if not model_proto_index: - return False - - variables_data_path = self.variables_data_path / component_name / model_alias - - for model_name, buffer_name in model_proto_index.items(): - if not (variables_data_path / model_name).is_file(): - return False - - return True - - def get_define_meta_from_file(self): - if not self.define_meta_path.is_file(): - return {} - return yaml.safe_load(self.define_meta_path.read_text('utf-8')) - - @DB.connection_context() - def get_define_meta_from_db(self, *query_args): - return tuple(PipelineComponentMeta.select().where(*self.query_args, *query_args)) - - def rearrange_define_meta(self, data): - define_meta = { - 'component_define': {}, - 'model_proto': {}, - } - - for row in data: - define_meta['component_define'][row.f_component_name] = {'module_name': row.f_component_module_name} - - # there is only one model_alias in a component - if row.f_component_name not in define_meta['model_proto']: - define_meta['model_proto'][row.f_component_name] = {} - define_meta['model_proto'][row.f_component_name][row.f_model_alias] = row.f_model_proto_index - - return define_meta - - def get_define_meta(self): - query = self.get_define_meta_from_db() - return self.rearrange_define_meta(query) if query else self.get_define_meta_from_file() - - @DB.connection_context() - def save_define_meta(self, component_name, component_module_name, model_alias, model_proto_index, run_parameters): - PipelineComponentMeta.insert( - f_model_id=self.model_id, - f_model_version=self.model_version, - f_role=self.role, - f_party_id=self.party_id, - f_component_name=component_name, - f_component_module_name=component_module_name, - f_model_alias=model_alias, - f_model_proto_index=model_proto_index, - f_run_parameters=run_parameters, - ).on_conflict(preserve=( - PipelineComponentMeta.f_update_time, - PipelineComponentMeta.f_update_date, - PipelineComponentMeta.f_component_module_name, - PipelineComponentMeta.f_model_alias, - PipelineComponentMeta.f_model_proto_index, - PipelineComponentMeta.f_run_parameters, - )).execute() - - @lock - def save_define_meta_from_db_to_file(self): - query = self.get_define_meta_from_db() - - for row in query: - run_parameters_path = self.get_run_parameters_path(row.f_component_name) - run_parameters_path.parent.mkdir(parents=True, exist_ok=True) - with run_parameters_path.open('w', encoding='utf-8') as f: - f.write(json_dumps(row.f_run_parameters)) - - self.define_meta_path.parent.mkdir(parents=True, exist_ok=True) - with self.define_meta_path.open('w', encoding='utf-8') as f: - yaml.dump(self.rearrange_define_meta(query), f, Dumper=yaml.RoundTripDumper) - - # import model - @local_cache_required(True) - def save_define_meta_from_file_to_db(self, replace_on_conflict=False): - if not replace_on_conflict: - with DB.connection_context(): - count = PipelineComponentMeta.select().where(*self.query_args).count() - if count > 0: - raise ValueError(f'The define_meta data already exists in database.') - - define_meta = self.get_define_meta_from_file() - run_parameters = self.get_run_parameters_from_files() - - insert = [] - for component_name, component_define in define_meta['component_define'].items(): - for model_alias, model_proto_index in define_meta['model_proto'][component_name].items(): - row = { - 'f_model_id': self.model_id, - 'f_model_version': self.model_version, - 'f_role': self.role, - 'f_party_id': self.party_id, - 'f_component_name': component_name, - 'f_component_module_name': component_define['module_name'], - 'f_model_alias': model_alias, - 'f_model_proto_index': model_proto_index, - 'f_run_parameters': run_parameters.get(component_name, {}), - } - insert.append(row) - - bulk_insert_into_db(PipelineComponentMeta, insert, replace_on_conflict) - - def replicate_define_meta(self, modification, query_args=(), replace_on_conflict=False): - query = self.get_define_meta_from_db(*query_args) - if not query: - return - - insert = [] - for row in query: - row = row.to_dict() - del row['id'] - row.update(modification) - insert.append(row) - - bulk_insert_into_db(PipelineComponentMeta, insert, replace_on_conflict) - - def get_run_parameters_path(self, component_name): - return self.run_parameters_path / component_name / 'run_parameters.json' - - @lock - def get_run_parameters_from_files(self): - if not self.run_parameters_path.is_dir(): - return {} - - return { - path.name: json_loads(self.get_run_parameters_path(path.name).read_text('utf-8')) - for path in self.run_parameters_path.iterdir() - } - - def get_run_parameters(self): - query = self.get_define_meta_from_db() - return { - row.f_component_name: row.f_run_parameters - for row in query - } if query else self.get_run_parameters_from_files() - - def get_archive_path(self, component_name): - return Path(TEMP_DIRECTORY, f'{self.party_model_id}_{self.model_version}_{component_name}.zip') - - def walk_component(self, zip_file, path: Path): - if path.is_dir(): - for subpath in path.iterdir(): - self.walk_component(zip_file, subpath) - elif path.is_file(): - zip_file.write(path, path.relative_to(self.model_path)) - - @local_cache_required(True) - def pack_component(self, component_name): - filename = self.get_archive_path(component_name) - - with ZipFile(filename, 'w') as zip_file: - self.walk_component(zip_file, self.variables_data_path / component_name) - self.walk_component(zip_file, self.checkpoint_path / component_name) - - hash_ = hashlib.sha256(filename.read_bytes()).hexdigest() - - return filename, hash_ - - @lock - def unpack_component(self, component_name, hash_=None): - filename = self.get_archive_path(component_name) - - if hash_ is not None: - sha256 = hashlib.sha256(filename.read_bytes()).hexdigest() - - if hash_ != sha256: - raise ValueError(f'Model archive hash mismatch. path: {filename} expected: {hash_} actual: {sha256}') - - with ZipFile(filename, 'r') as zip_file: - zip_file.extractall(self.model_path) diff --git a/python/fate_flow/pipelined_model/pipelined_model.py b/python/fate_flow/pipelined_model/pipelined_model.py deleted file mode 100644 index 351a71a26..000000000 --- a/python/fate_flow/pipelined_model/pipelined_model.py +++ /dev/null @@ -1,318 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import base64 -import hashlib -import json -import os -import shutil -import typing - -from google.protobuf import json_format - -from fate_arch.common.base_utils import json_dumps, json_loads - -from fate_flow.component_env_utils import provider_utils -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.model import ( - Locker, local_cache_required, - lock, parse_proto_object, -) -from fate_flow.pipelined_model.pipelined_component import PipelinedComponent -from fate_flow.protobuf.python.pipeline_pb2 import Pipeline -from fate_flow.settings import TEMP_DIRECTORY, stat_logger -from fate_flow.utils.job_utils import ( - PIPELINE_COMPONENT_NAME, PIPELINE_MODEL_ALIAS, - PIPELINE_COMPONENT_MODULE_NAME, PIPELINE_MODEL_NAME, -) -from fate_flow.utils.base_utils import get_fate_flow_directory - - -class PipelinedModel(Locker): - def __init__(self, model_id, model_version): - """ - Support operations on FATE PipelinedModels - :param model_id: the model id stored at the local party. - :param model_version: the model version. - """ - os.makedirs(TEMP_DIRECTORY, exist_ok=True) - - self.role, self.party_id, self._model_id = model_id.split('#', 2) - self.party_model_id = self.model_id = model_id - self.model_version = model_version - - self.pipelined_component = PipelinedComponent(role=self.role, party_id=self.party_id, - model_id=self._model_id, model_version=self.model_version) - self.model_path = self.pipelined_component.model_path - - super().__init__(self.model_path) - - def save_pipeline_model(self, pipeline_buffer_object, save_define_meta_file=True): - model_buffers = { - PIPELINE_MODEL_NAME: ( - type(pipeline_buffer_object).__name__, - pipeline_buffer_object.SerializeToString(), - json_format.MessageToDict(pipeline_buffer_object, including_default_value_fields=True), - ), - } - self.save_component_model(PIPELINE_COMPONENT_NAME, PIPELINE_COMPONENT_MODULE_NAME, PIPELINE_MODEL_ALIAS, model_buffers) - - # only update pipeline model file if save_define_meta_file is False - if save_define_meta_file: - self.pipelined_component.save_define_meta_from_db_to_file() - - def save_component_model(self, *args, **kwargs): - component_model = self.create_component_model(*args, **kwargs) - self.write_component_model(component_model) - - def create_component_model(self, component_name, component_module_name, model_alias, - model_buffers: typing.Dict[str, typing.Tuple[str, bytes, dict]], - user_specified_run_parameters: dict = None): - component_model = {"buffer": {}} - - component_model_storage_path = os.path.join(self.pipelined_component.variables_data_path, component_name, model_alias) - model_proto_index = {} - - for model_name, (proto_index, object_serialized, object_json) in model_buffers.items(): - storage_path = os.path.join(component_model_storage_path, model_name).replace(get_fate_flow_directory(), "") - component_model["buffer"][storage_path] = (base64.b64encode(object_serialized).decode(), object_json) - model_proto_index[model_name] = proto_index # index of model name and proto buffer class name - - stat_logger.info(f"saved {component_name} {model_alias} {model_name} buffer") - - component_model["component_name"] = component_name - component_model["component_module_name"] = component_module_name - component_model["model_alias"] = model_alias - component_model["model_proto_index"] = model_proto_index - component_model["run_parameters"] = user_specified_run_parameters - - return component_model - - @lock - def write_component_model(self, component_model): - for storage_path, (object_serialized_encoded, object_json) in component_model.get("buffer").items(): - storage_path = get_fate_flow_directory() + storage_path - os.makedirs(os.path.dirname(storage_path), exist_ok=True) - - with open(storage_path, "wb") as fw: - fw.write(base64.b64decode(object_serialized_encoded.encode())) - - with open(f"{storage_path}.json", "w", encoding="utf8") as fw: - fw.write(json_dumps(object_json)) - - self.pipelined_component.save_define_meta( - component_model["component_name"], component_model["component_module_name"], - component_model["model_alias"], component_model["model_proto_index"], - component_model.get("run_parameters") or {}, - ) - - stat_logger.info(f'saved {component_model["component_name"]} {component_model["model_alias"]} successfully') - - @local_cache_required(True) - def _read_component_model(self, component_name, model_alias): - component_model_storage_path = os.path.join(self.pipelined_component.variables_data_path, component_name, model_alias) - model_proto_index = self.get_model_proto_index(component_name=component_name, model_alias=model_alias) - - model_buffers = {} - for model_name, buffer_name in model_proto_index.items(): - storage_path = os.path.join(component_model_storage_path, model_name) - - with open(storage_path, "rb") as f: - buffer_object_serialized_string = f.read() - - try: - with open(f"{storage_path}.json", encoding="utf-8") as f: - buffer_object_json_format = json_loads(f.read()) - except FileNotFoundError: - buffer_object_json_format = "" - # TODO: should be running in worker - """ - buffer_object_json_format = json_format.MessageToDict( - parse_proto_object(buffer_name, buffer_object_serialized_string), - including_default_value_fields=True - ) - with open(f"{storage_path}.json", "x", encoding="utf-8") as f: - f.write(json_dumps(buffer_object_json_format)) - """ - - model_buffers[model_name] = ( - buffer_name, - buffer_object_serialized_string, - buffer_object_json_format, - ) - - return model_buffers - - # TODO: use different functions instead of passing arguments - def read_component_model(self, component_name, model_alias=None, parse=True, output_json=False): - if model_alias is None: - model_alias = self.get_model_alias(component_name) - - if not self.pipelined_component.exists(component_name, model_alias): - return {} - - _model_buffers = self._read_component_model(component_name, model_alias) - - model_buffers = {} - for model_name, ( - buffer_name, - buffer_object_serialized_string, - buffer_object_json_format, - ) in _model_buffers.items(): - if output_json: - model_buffers[model_name] = buffer_object_json_format - elif parse: - model_buffers[model_name] = parse_proto_object(buffer_name, buffer_object_serialized_string) - else: - model_buffers[model_name] = [ - buffer_name, - base64.b64encode(buffer_object_serialized_string).decode(), - ] - - return model_buffers - - # TODO: integration with read_component_model - @local_cache_required(True) - def read_pipeline_model(self, parse=True): - component_model_storage_path = os.path.join(self.pipelined_component.variables_data_path, PIPELINE_COMPONENT_NAME, PIPELINE_MODEL_ALIAS) - model_proto_index = self.get_model_proto_index(PIPELINE_COMPONENT_NAME, PIPELINE_MODEL_ALIAS) - - model_buffers = {} - for model_name, buffer_name in model_proto_index.items(): - with open(os.path.join(component_model_storage_path, model_name), "rb") as fr: - buffer_object_serialized_string = fr.read() - - model_buffers[model_name] = (parse_proto_object(buffer_name, buffer_object_serialized_string, Pipeline) if parse - else [buffer_name, base64.b64encode(buffer_object_serialized_string).decode()]) - - return model_buffers[PIPELINE_MODEL_NAME] - - @local_cache_required(True) - def collect_models(self, in_bytes=False, b64encode=True): - define_meta = self.pipelined_component.get_define_meta() - model_buffers = {} - - for component_name in define_meta.get("model_proto", {}).keys(): - for model_alias, model_proto_index in define_meta["model_proto"][component_name].items(): - component_model_storage_path = os.path.join(self.pipelined_component.variables_data_path, component_name, model_alias) - - for model_name, buffer_name in model_proto_index.items(): - with open(os.path.join(component_model_storage_path, model_name), "rb") as fr: - serialized_string = fr.read() - - if in_bytes: - if b64encode: - serialized_string = base64.b64encode(serialized_string).decode() - - model_buffers[f"{component_name}.{model_alias}:{model_name}"] = serialized_string - else: - model_buffers[model_name] = parse_proto_object(buffer_name, serialized_string) - - return model_buffers - - @staticmethod - def get_model_migrate_tool(): - return provider_utils.get_provider_class_object(RuntimeConfig.COMPONENT_PROVIDER, "model_migrate", True) - - @staticmethod - def get_homo_model_convert_tool(): - return provider_utils.get_provider_class_object(RuntimeConfig.COMPONENT_PROVIDER, "homo_model_convert", True) - - def exists(self): - return self.pipelined_component.exists() - - @local_cache_required(True) - def packaging_model(self): - self.gen_model_import_config() - - # self.archive_model_file_path - shutil.make_archive(self.archive_model_base_path, 'zip', self.model_path) - - with open(self.archive_model_file_path, 'rb') as f: - hash_ = hashlib.sha256(f.read()).hexdigest() - - stat_logger.info(f'Make model {self.model_id} {self.model_version} archive successfully. ' - f'path: {self.archive_model_file_path} hash: {hash_}') - return hash_ - - @lock - def unpack_model(self, archive_file_path: str, force_update: bool = False, hash_: str = None): - if self.exists() and not force_update: - raise FileExistsError(f'Model {self.model_id} {self.model_version} local cache already existed.') - - if hash_ is not None: - with open(archive_file_path, 'rb') as f: - sha256 = hashlib.sha256(f.read()).hexdigest() - - if hash_ != sha256: - raise ValueError(f'Model archive hash mismatch. ' - f'path: {archive_file_path} expected: {hash_} actual: {sha256}') - - shutil.unpack_archive(archive_file_path, self.model_path, 'zip') - - stat_logger.info(f'Unpack model {self.model_id} {self.model_version} archive successfully. path: {self.model_path}') - - def get_component_define(self, component_name=None): - component_define = self.pipelined_component.get_define_meta()['component_define'] - if component_name is None: - return component_define - - return component_define.get(component_name, {}) - - def get_model_proto_index(self, component_name=None, model_alias=None): - model_proto = self.pipelined_component.get_define_meta()['model_proto'] - if component_name is None: - return model_proto - - model_proto = model_proto.get(component_name, {}) - if model_alias is None: - return model_proto - - return model_proto.get(model_alias, {}) - - def get_model_alias(self, component_name): - model_proto_index = self.get_model_proto_index(component_name) - if len(model_proto_index) != 1: - raise KeyError('Failed to detect "model_alias", please specify it manually.') - - return next(iter(model_proto_index.keys())) - - @property - def archive_model_base_path(self): - return os.path.join(TEMP_DIRECTORY, f'{self.party_model_id}_{self.model_version}') - - @property - def archive_model_file_path(self): - return f'{self.archive_model_base_path}.zip' - - @local_cache_required(True) - def calculate_model_file_size(self): - size = 0 - for root, dirs, files in os.walk(self.model_path): - size += sum([os.path.getsize(os.path.join(root, name)) for name in files]) - return round(size/1024) - - @local_cache_required(True) - def gen_model_import_config(self): - config = { - 'role': self.role, - 'party_id': int(self.party_id), - 'model_id': self._model_id, - 'model_version': self.model_version, - 'file': self.archive_model_file_path, - 'force_update': False, - } - with (self.model_path / 'import_model.json').open('w', encoding='utf-8') as f: - json.dump(config, f, indent=4) diff --git a/python/fate_flow/pipelined_model/publish_model.py b/python/fate_flow/pipelined_model/publish_model.py deleted file mode 100644 index 1ffdbefc2..000000000 --- a/python/fate_flow/pipelined_model/publish_model.py +++ /dev/null @@ -1,232 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os - -import grpc - -from fate_arch.common.base_utils import json_loads -from fate_arch.protobuf.python import model_service_pb2, model_service_pb2_grpc - -from fate_flow.model.sync_model import SyncModel -from fate_flow.pipelined_model import pipelined_model -from fate_flow.pipelined_model.homo_model_deployer.model_deploy import model_deploy -from fate_flow.settings import ( - ENABLE_MODEL_STORE, FATE_FLOW_MODEL_TRANSFER_ENDPOINT, - GRPC_OPTIONS, HOST, HTTP_PORT, USE_REGISTRY, stat_logger, -) -from fate_flow.utils import model_utils - - -def generate_publish_model_info(config_data): - model_id = config_data['job_parameters']['model_id'] - model_version = config_data['job_parameters']['model_version'] - config_data['model'] = {} - for role, role_party in config_data.get("role").items(): - config_data['model'][role] = {} - for party_id in role_party: - config_data['model'][role][party_id] = { - 'model_id': model_utils.gen_party_model_id(model_id, role, party_id), - 'model_version': model_version - } - - -def load_model(config_data): - stat_logger.info(config_data) - if not config_data.get('servings'): - return 100, 'Please configure servings address' - - for serving in config_data['servings']: - with grpc.insecure_channel(serving, GRPC_OPTIONS) as channel: - stub = model_service_pb2_grpc.ModelServiceStub(channel) - load_model_request = model_service_pb2.PublishRequest() - for role_name, role_partys in config_data.get("role", {}).items(): - for _party_id in role_partys: - load_model_request.role[role_name].partyId.append(str(_party_id)) - for role_name, role_model_config in config_data.get("model", {}).items(): - for _party_id, role_party_model_config in role_model_config.items(): - load_model_request.model[role_name].roleModelInfo[str(_party_id)].tableName = \ - role_party_model_config['model_version'] - load_model_request.model[role_name].roleModelInfo[str(_party_id)].namespace = \ - role_party_model_config['model_id'] - - stat_logger.info('request serving: {} load model'.format(serving)) - load_model_request.local.role = config_data.get('local', {}).get('role', '') - load_model_request.local.partyId = str(config_data.get('local', {}).get('party_id', '')) - load_model_request.loadType = config_data['job_parameters'].get("load_type", "FATEFLOW") - # make use of 'model.transfer.url' in serving server - use_serving_url = config_data['job_parameters'].get('use_transfer_url_on_serving', False) - if not USE_REGISTRY and not use_serving_url: - load_model_request.filePath = f"http://{HOST}:{HTTP_PORT}{FATE_FLOW_MODEL_TRANSFER_ENDPOINT}" - else: - load_model_request.filePath = config_data['job_parameters'].get("file_path", "") - stat_logger.info(load_model_request) - response = stub.publishLoad(load_model_request) - stat_logger.info( - '{} {} load model status: {}'.format(load_model_request.local.role, load_model_request.local.partyId, - response.statusCode)) - if response.statusCode != 0: - return response.statusCode, '{} {}'.format(response.message, response.error) - return 0, 'success' - - -def bind_model_service(config_data): - if not config_data.get('servings'): - return 100, 'Please configure servings address' - - service_id = str(config_data.get('service_id', '')) - initiator_role = config_data['initiator']['role'] - initiator_party_id = str(config_data['initiator']['party_id']) - model_id = config_data['job_parameters']['model_id'] - model_version = config_data['job_parameters']['model_version'] - - for serving in config_data['servings']: - with grpc.insecure_channel(serving, GRPC_OPTIONS) as channel: - stub = model_service_pb2_grpc.ModelServiceStub(channel) - publish_model_request = model_service_pb2.PublishRequest() - publish_model_request.serviceId = service_id - - # {"role": {"guest": ["9999"], "host": ["10000"], "arbiter": ["9999"]}} - for role_name, role_party in config_data.get("role").items(): - publish_model_request.role[role_name].partyId.extend([str(party_id) for party_id in role_party]) - - party_model_id = model_utils.gen_party_model_id(model_id, initiator_role, initiator_party_id) - publish_model_request.model[initiator_role].roleModelInfo[initiator_party_id].tableName = model_version - publish_model_request.model[initiator_role].roleModelInfo[initiator_party_id].namespace = party_model_id - - publish_model_request.local.role = initiator_role - publish_model_request.local.partyId = initiator_party_id - - stat_logger.info(publish_model_request) - response = stub.publishBind(publish_model_request) - stat_logger.info(response) - if response.statusCode != 0: - return response.statusCode, response.message - - return 0, None - - -def download_model(party_model_id, model_version): - if ENABLE_MODEL_STORE: - sync_model = SyncModel( - party_model_id=party_model_id, - model_version=model_version, - ) - if sync_model.remote_exists(): - sync_model.download(True) - - model = pipelined_model.PipelinedModel(party_model_id, model_version) - if not model.exists(): - return {} - return model.collect_models(in_bytes=True) - - -def convert_homo_model(request_data): - party_model_id = model_utils.gen_party_model_id(model_id=request_data["model_id"], - role=request_data["role"], - party_id=request_data["party_id"]) - model_version = request_data.get("model_version") - model = pipelined_model.PipelinedModel(model_id=party_model_id, model_version=model_version) - if not model.exists(): - return 100, 'Model {} {} does not exist'.format(party_model_id, model_version), None - - define_meta = model.pipelined_component.get_define_meta() - - framework_name = request_data.get("framework_name") - detail = [] - # todo: use subprocess? - convert_tool = model.get_homo_model_convert_tool() - for key, value in define_meta.get("model_proto", {}).items(): - if key == 'pipeline': - continue - for model_alias in value.keys(): - buffer_obj = model.read_component_model(key, model_alias) - module_name = define_meta.get("component_define", {}).get(key, {}).get('module_name') - converted_framework, converted_model = convert_tool.model_convert(model_contents=buffer_obj, - module_name=module_name, - framework_name=framework_name) - if converted_model: - converted_model_dir = os.path.join(model.pipelined_component.variables_data_path, key, model_alias, "converted_model") - os.makedirs(converted_model_dir, exist_ok=True) - - saved_path = convert_tool.save_converted_model(converted_model, - converted_framework, - converted_model_dir) - detail.append({ - "component_name": key, - "model_alias": model_alias, - "converted_model_path": saved_path - }) - if len(detail) > 0: - return (0, - f"Conversion of homogeneous federated learning component(s) in model " - f"{party_model_id}:{model_version} completed. Use export or homo/deploy " - f"to download or deploy the converted model.", - detail) - else: - return 100, f"No component in model {party_model_id}:{model_version} can be converted.", None - - -def deploy_homo_model(request_data): - party_model_id = model_utils.gen_party_model_id(model_id=request_data["model_id"], - role=request_data["role"], - party_id=request_data["party_id"]) - model_version = request_data["model_version"] - component_name = request_data['component_name'] - service_id = request_data['service_id'] - framework_name = request_data.get('framework_name') - model = pipelined_model.PipelinedModel(model_id=party_model_id, model_version=model_version) - if not model.exists(): - return 100, 'Model {} {} does not exist'.format(party_model_id, model_version), None - - # get the model alias from the dsl saved with the pipeline - pipeline = model.read_pipeline_model() - train_dsl = json_loads(pipeline.train_dsl) - if component_name not in train_dsl.get('components', {}): - return 100, 'Model {} {} does not contain component {}'.\ - format(party_model_id, model_version, component_name), None - - model_alias_list = train_dsl['components'][component_name].get('output', {}).get('model') - if not model_alias_list: - return 100, 'Component {} in Model {} {} does not have output model'. \ - format(component_name, party_model_id, model_version), None - - # currently there is only one model output - model_alias = model_alias_list[0] - converted_model_dir = os.path.join(model.pipelined_component.variables_data_path, component_name, model_alias, "converted_model") - if not os.path.isdir(converted_model_dir): - return 100, '''Component {} in Model {} {} isn't converted'''.\ - format(component_name, party_model_id, model_version), None - - # todo: use subprocess? - convert_tool = model.get_homo_model_convert_tool() - if not framework_name: - module_name = train_dsl['components'][component_name].get('module') - buffer_obj = model.read_component_model(component_name, model_alias) - framework_name = convert_tool.get_default_target_framework(model_contents=buffer_obj, module_name=module_name) - - model_object = convert_tool.load_converted_model(base_dir=converted_model_dir, - framework_name=framework_name) - deployed_service = model_deploy(party_model_id, - model_version, - model_object, - framework_name, - service_id, - request_data['deployment_type'], - request_data['deployment_parameters']) - return (0, - f"An online serving service is started in the {request_data['deployment_type']} system.", - deployed_service) - diff --git a/python/fate_flow/proto/__init__.py b/python/fate_flow/proto/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/proto/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/proto/rollsite/__init__.py b/python/fate_flow/proto/rollsite/__init__.py new file mode 100644 index 000000000..f958736b0 --- /dev/null +++ b/python/fate_flow/proto/rollsite/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys + +_pb_path = os.path.abspath(os.path.join(__file__, os.path.pardir)) +if _pb_path not in sys.path: + sys.path.append(_pb_path) \ No newline at end of file diff --git a/python/fate_flow/proto/rollsite/basic_meta_pb2.py b/python/fate_flow/proto/rollsite/basic_meta_pb2.py new file mode 100644 index 000000000..ca81e5d1f --- /dev/null +++ b/python/fate_flow/proto/rollsite/basic_meta_pb2.py @@ -0,0 +1,209 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: basic-meta.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x10\x62\x61sic-meta.proto\x12\x1e\x63om.webank.ai.eggroll.api.core"6\n\x08\x45ndpoint\x12\n\n\x02ip\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\x12\x10\n\x08hostname\x18\x03 \x01(\t"H\n\tEndpoints\x12;\n\tendpoints\x18\x01 \x03(\x0b\x32(.com.webank.ai.eggroll.api.core.Endpoint"H\n\x04\x44\x61ta\x12\x0e\n\x06isNull\x18\x01 \x01(\x08\x12\x14\n\x0chostLanguage\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c"F\n\x0cRepeatedData\x12\x36\n\x08\x64\x61talist\x18\x01 \x03(\x0b\x32$.com.webank.ai.eggroll.api.core.Data"u\n\x0b\x43\x61llRequest\x12\x0f\n\x07isAsync\x18\x01 \x01(\x08\x12\x0f\n\x07timeout\x18\x02 \x01(\x03\x12\x0f\n\x07\x63ommand\x18\x03 \x01(\t\x12\x33\n\x05param\x18\x04 \x01(\x0b\x32$.com.webank.ai.eggroll.api.core.Data"\x88\x01\n\x0c\x43\x61llResponse\x12\x42\n\x0creturnStatus\x18\x01 \x01(\x0b\x32,.com.webank.ai.eggroll.api.core.ReturnStatus\x12\x34\n\x06result\x18\x02 \x01(\x0b\x32$.com.webank.ai.eggroll.api.core.Data""\n\x03Job\x12\r\n\x05jobId\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t"Y\n\x04Task\x12\x30\n\x03job\x18\x01 \x01(\x0b\x32#.com.webank.ai.eggroll.api.core.Job\x12\x0e\n\x06taskId\x18\x02 \x01(\x03\x12\x0f\n\x07tableId\x18\x03 \x01(\x03"N\n\x06Result\x12\x32\n\x04task\x18\x01 \x01(\x0b\x32$.com.webank.ai.eggroll.api.core.Task\x12\x10\n\x08resultId\x18\x02 \x01(\x03"-\n\x0cReturnStatus\x12\x0c\n\x04\x63ode\x18\x01 \x01(\x05\x12\x0f\n\x07message\x18\x02 \x01(\t"\xe2\x01\n\x0bSessionInfo\x12\x11\n\tsessionId\x18\x01 \x01(\t\x12\x61\n\x13\x63omputingEngineConf\x18\x02 \x03(\x0b\x32\x44.com.webank.ai.eggroll.api.core.SessionInfo.ComputingEngineConfEntry\x12\x14\n\x0cnamingPolicy\x18\x03 \x01(\t\x12\x0b\n\x03tag\x18\x04 \x01(\t\x1a:\n\x18\x43omputingEngineConfEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x62\x06proto3' +) + + +_ENDPOINT = DESCRIPTOR.message_types_by_name["Endpoint"] +_ENDPOINTS = DESCRIPTOR.message_types_by_name["Endpoints"] +_DATA = DESCRIPTOR.message_types_by_name["Data"] +_REPEATEDDATA = DESCRIPTOR.message_types_by_name["RepeatedData"] +_CALLREQUEST = DESCRIPTOR.message_types_by_name["CallRequest"] +_CALLRESPONSE = DESCRIPTOR.message_types_by_name["CallResponse"] +_JOB = DESCRIPTOR.message_types_by_name["Job"] +_TASK = DESCRIPTOR.message_types_by_name["Task"] +_RESULT = DESCRIPTOR.message_types_by_name["Result"] +_RETURNSTATUS = DESCRIPTOR.message_types_by_name["ReturnStatus"] +_SESSIONINFO = DESCRIPTOR.message_types_by_name["SessionInfo"] +_SESSIONINFO_COMPUTINGENGINECONFENTRY = _SESSIONINFO.nested_types_by_name[ + "ComputingEngineConfEntry" +] +Endpoint = _reflection.GeneratedProtocolMessageType( + "Endpoint", + (_message.Message,), + { + "DESCRIPTOR": _ENDPOINT, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Endpoint) + }, +) +_sym_db.RegisterMessage(Endpoint) + +Endpoints = _reflection.GeneratedProtocolMessageType( + "Endpoints", + (_message.Message,), + { + "DESCRIPTOR": _ENDPOINTS, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Endpoints) + }, +) +_sym_db.RegisterMessage(Endpoints) + +Data = _reflection.GeneratedProtocolMessageType( + "Data", + (_message.Message,), + { + "DESCRIPTOR": _DATA, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Data) + }, +) +_sym_db.RegisterMessage(Data) + +RepeatedData = _reflection.GeneratedProtocolMessageType( + "RepeatedData", + (_message.Message,), + { + "DESCRIPTOR": _REPEATEDDATA, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.RepeatedData) + }, +) +_sym_db.RegisterMessage(RepeatedData) + +CallRequest = _reflection.GeneratedProtocolMessageType( + "CallRequest", + (_message.Message,), + { + "DESCRIPTOR": _CALLREQUEST, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.CallRequest) + }, +) +_sym_db.RegisterMessage(CallRequest) + +CallResponse = _reflection.GeneratedProtocolMessageType( + "CallResponse", + (_message.Message,), + { + "DESCRIPTOR": _CALLRESPONSE, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.CallResponse) + }, +) +_sym_db.RegisterMessage(CallResponse) + +Job = _reflection.GeneratedProtocolMessageType( + "Job", + (_message.Message,), + { + "DESCRIPTOR": _JOB, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Job) + }, +) +_sym_db.RegisterMessage(Job) + +Task = _reflection.GeneratedProtocolMessageType( + "Task", + (_message.Message,), + { + "DESCRIPTOR": _TASK, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Task) + }, +) +_sym_db.RegisterMessage(Task) + +Result = _reflection.GeneratedProtocolMessageType( + "Result", + (_message.Message,), + { + "DESCRIPTOR": _RESULT, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Result) + }, +) +_sym_db.RegisterMessage(Result) + +ReturnStatus = _reflection.GeneratedProtocolMessageType( + "ReturnStatus", + (_message.Message,), + { + "DESCRIPTOR": _RETURNSTATUS, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.ReturnStatus) + }, +) +_sym_db.RegisterMessage(ReturnStatus) + +SessionInfo = _reflection.GeneratedProtocolMessageType( + "SessionInfo", + (_message.Message,), + { + "ComputingEngineConfEntry": _reflection.GeneratedProtocolMessageType( + "ComputingEngineConfEntry", + (_message.Message,), + { + "DESCRIPTOR": _SESSIONINFO_COMPUTINGENGINECONFENTRY, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.SessionInfo.ComputingEngineConfEntry) + }, + ), + "DESCRIPTOR": _SESSIONINFO, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.SessionInfo) + }, +) +_sym_db.RegisterMessage(SessionInfo) +_sym_db.RegisterMessage(SessionInfo.ComputingEngineConfEntry) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _SESSIONINFO_COMPUTINGENGINECONFENTRY._options = None + _SESSIONINFO_COMPUTINGENGINECONFENTRY._serialized_options = b"8\001" + _ENDPOINT._serialized_start = 52 + _ENDPOINT._serialized_end = 106 + _ENDPOINTS._serialized_start = 108 + _ENDPOINTS._serialized_end = 180 + _DATA._serialized_start = 182 + _DATA._serialized_end = 254 + _REPEATEDDATA._serialized_start = 256 + _REPEATEDDATA._serialized_end = 326 + _CALLREQUEST._serialized_start = 328 + _CALLREQUEST._serialized_end = 445 + _CALLRESPONSE._serialized_start = 448 + _CALLRESPONSE._serialized_end = 584 + _JOB._serialized_start = 586 + _JOB._serialized_end = 620 + _TASK._serialized_start = 622 + _TASK._serialized_end = 711 + _RESULT._serialized_start = 713 + _RESULT._serialized_end = 791 + _RETURNSTATUS._serialized_start = 793 + _RETURNSTATUS._serialized_end = 838 + _SESSIONINFO._serialized_start = 841 + _SESSIONINFO._serialized_end = 1067 + _SESSIONINFO_COMPUTINGENGINECONFENTRY._serialized_start = 1009 + _SESSIONINFO_COMPUTINGENGINECONFENTRY._serialized_end = 1067 +# @@protoc_insertion_point(module_scope) diff --git a/python/fate_flow/proto/rollsite/proxy_pb2.py b/python/fate_flow/proto/rollsite/proxy_pb2.py new file mode 100644 index 000000000..3bbd35d6a --- /dev/null +++ b/python/fate_flow/proto/rollsite/proxy_pb2.py @@ -0,0 +1,197 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: proxy.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import enum_type_wrapper + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x0bproxy.proto\x12*com.webank.ai.eggroll.api.networking.proxy\x1a\x10\x62\x61sic-meta.proto"&\n\x05Model\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07\x64\x61taKey\x18\x02 \x01(\t"X\n\x04Task\x12\x0e\n\x06taskId\x18\x01 \x01(\t\x12@\n\x05model\x18\x02 \x01(\x0b\x32\x31.com.webank.ai.eggroll.api.networking.proxy.Model"p\n\x05Topic\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07partyId\x18\x02 \x01(\t\x12\x0c\n\x04role\x18\x03 \x01(\t\x12:\n\x08\x63\x61llback\x18\x04 \x01(\x0b\x32(.com.webank.ai.eggroll.api.core.Endpoint"\x17\n\x07\x43ommand\x12\x0c\n\x04name\x18\x01 \x01(\t"p\n\x04\x43onf\x12\x16\n\x0eoverallTimeout\x18\x01 \x01(\x03\x12\x1d\n\x15\x63ompletionWaitTimeout\x18\x02 \x01(\x03\x12\x1d\n\x15packetIntervalTimeout\x18\x03 \x01(\x03\x12\x12\n\nmaxRetries\x18\x04 \x01(\x05"\x9a\x03\n\x08Metadata\x12>\n\x04task\x18\x01 \x01(\x0b\x32\x30.com.webank.ai.eggroll.api.networking.proxy.Task\x12>\n\x03src\x18\x02 \x01(\x0b\x32\x31.com.webank.ai.eggroll.api.networking.proxy.Topic\x12>\n\x03\x64st\x18\x03 \x01(\x0b\x32\x31.com.webank.ai.eggroll.api.networking.proxy.Topic\x12\x44\n\x07\x63ommand\x18\x04 \x01(\x0b\x32\x33.com.webank.ai.eggroll.api.networking.proxy.Command\x12\x10\n\x08operator\x18\x05 \x01(\t\x12\x0b\n\x03seq\x18\x06 \x01(\x03\x12\x0b\n\x03\x61\x63k\x18\x07 \x01(\x03\x12>\n\x04\x63onf\x18\x08 \x01(\x0b\x32\x30.com.webank.ai.eggroll.api.networking.proxy.Conf\x12\x0b\n\x03\x65xt\x18\t \x01(\x0c\x12\x0f\n\x07version\x18\x64 \x01(\t""\n\x04\x44\x61ta\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c"\x8e\x01\n\x06Packet\x12\x44\n\x06header\x18\x01 \x01(\x0b\x32\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata\x12>\n\x04\x62ody\x18\x02 \x01(\x0b\x32\x30.com.webank.ai.eggroll.api.networking.proxy.Data"\xa3\x01\n\x11HeartbeatResponse\x12\x44\n\x06header\x18\x01 \x01(\x0b\x32\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata\x12H\n\toperation\x18\x02 \x01(\x0e\x32\x35.com.webank.ai.eggroll.api.networking.proxy.Operation"\xc5\x01\n\x0cPollingFrame\x12\x0e\n\x06method\x18\x01 \x01(\t\x12\x0b\n\x03seq\x18\x02 \x01(\x03\x12\x46\n\x08metadata\x18\n \x01(\x0b\x32\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata\x12\x42\n\x06packet\x18\x14 \x01(\x0b\x32\x32.com.webank.ai.eggroll.api.networking.proxy.Packet\x12\x0c\n\x04\x64\x65sc\x18\x1e \x01(\t*O\n\tOperation\x12\t\n\x05START\x10\x00\x12\x07\n\x03RUN\x10\x01\x12\x08\n\x04STOP\x10\x02\x12\x08\n\x04KILL\x10\x03\x12\x0c\n\x08GET_DATA\x10\x04\x12\x0c\n\x08PUT_DATA\x10\x05\x32\xf6\x03\n\x13\x44\x61taTransferService\x12r\n\x04push\x12\x32.com.webank.ai.eggroll.api.networking.proxy.Packet\x1a\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata(\x01\x12r\n\x04pull\x12\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata\x1a\x32.com.webank.ai.eggroll.api.networking.proxy.Packet0\x01\x12s\n\tunaryCall\x12\x32.com.webank.ai.eggroll.api.networking.proxy.Packet\x1a\x32.com.webank.ai.eggroll.api.networking.proxy.Packet\x12\x81\x01\n\x07polling\x12\x38.com.webank.ai.eggroll.api.networking.proxy.PollingFrame\x1a\x38.com.webank.ai.eggroll.api.networking.proxy.PollingFrame(\x01\x30\x01\x32t\n\x0cRouteService\x12\x64\n\x05query\x12\x31.com.webank.ai.eggroll.api.networking.proxy.Topic\x1a(.com.webank.ai.eggroll.api.core.Endpointb\x06proto3' +) + +_OPERATION = DESCRIPTOR.enum_types_by_name["Operation"] +Operation = enum_type_wrapper.EnumTypeWrapper(_OPERATION) +START = 0 +RUN = 1 +STOP = 2 +KILL = 3 +GET_DATA = 4 +PUT_DATA = 5 + + +_MODEL = DESCRIPTOR.message_types_by_name["Model"] +_TASK = DESCRIPTOR.message_types_by_name["Task"] +_TOPIC = DESCRIPTOR.message_types_by_name["Topic"] +_COMMAND = DESCRIPTOR.message_types_by_name["Command"] +_CONF = DESCRIPTOR.message_types_by_name["Conf"] +_METADATA = DESCRIPTOR.message_types_by_name["Metadata"] +_DATA = DESCRIPTOR.message_types_by_name["Data"] +_PACKET = DESCRIPTOR.message_types_by_name["Packet"] +_HEARTBEATRESPONSE = DESCRIPTOR.message_types_by_name["HeartbeatResponse"] +_POLLINGFRAME = DESCRIPTOR.message_types_by_name["PollingFrame"] +Model = _reflection.GeneratedProtocolMessageType( + "Model", + (_message.Message,), + { + "DESCRIPTOR": _MODEL, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Model) + }, +) +_sym_db.RegisterMessage(Model) + +Task = _reflection.GeneratedProtocolMessageType( + "Task", + (_message.Message,), + { + "DESCRIPTOR": _TASK, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Task) + }, +) +_sym_db.RegisterMessage(Task) + +Topic = _reflection.GeneratedProtocolMessageType( + "Topic", + (_message.Message,), + { + "DESCRIPTOR": _TOPIC, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Topic) + }, +) +_sym_db.RegisterMessage(Topic) + +Command = _reflection.GeneratedProtocolMessageType( + "Command", + (_message.Message,), + { + "DESCRIPTOR": _COMMAND, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Command) + }, +) +_sym_db.RegisterMessage(Command) + +Conf = _reflection.GeneratedProtocolMessageType( + "Conf", + (_message.Message,), + { + "DESCRIPTOR": _CONF, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Conf) + }, +) +_sym_db.RegisterMessage(Conf) + +Metadata = _reflection.GeneratedProtocolMessageType( + "Metadata", + (_message.Message,), + { + "DESCRIPTOR": _METADATA, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Metadata) + }, +) +_sym_db.RegisterMessage(Metadata) + +Data = _reflection.GeneratedProtocolMessageType( + "Data", + (_message.Message,), + { + "DESCRIPTOR": _DATA, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Data) + }, +) +_sym_db.RegisterMessage(Data) + +Packet = _reflection.GeneratedProtocolMessageType( + "Packet", + (_message.Message,), + { + "DESCRIPTOR": _PACKET, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Packet) + }, +) +_sym_db.RegisterMessage(Packet) + +HeartbeatResponse = _reflection.GeneratedProtocolMessageType( + "HeartbeatResponse", + (_message.Message,), + { + "DESCRIPTOR": _HEARTBEATRESPONSE, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.HeartbeatResponse) + }, +) +_sym_db.RegisterMessage(HeartbeatResponse) + +PollingFrame = _reflection.GeneratedProtocolMessageType( + "PollingFrame", + (_message.Message,), + { + "DESCRIPTOR": _POLLINGFRAME, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.PollingFrame) + }, +) +_sym_db.RegisterMessage(PollingFrame) + +_DATATRANSFERSERVICE = DESCRIPTOR.services_by_name["DataTransferService"] +_ROUTESERVICE = DESCRIPTOR.services_by_name["RouteService"] +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _OPERATION._serialized_start = 1420 + _OPERATION._serialized_end = 1499 + _MODEL._serialized_start = 77 + _MODEL._serialized_end = 115 + _TASK._serialized_start = 117 + _TASK._serialized_end = 205 + _TOPIC._serialized_start = 207 + _TOPIC._serialized_end = 319 + _COMMAND._serialized_start = 321 + _COMMAND._serialized_end = 344 + _CONF._serialized_start = 346 + _CONF._serialized_end = 458 + _METADATA._serialized_start = 461 + _METADATA._serialized_end = 871 + _DATA._serialized_start = 873 + _DATA._serialized_end = 907 + _PACKET._serialized_start = 910 + _PACKET._serialized_end = 1052 + _HEARTBEATRESPONSE._serialized_start = 1055 + _HEARTBEATRESPONSE._serialized_end = 1218 + _POLLINGFRAME._serialized_start = 1221 + _POLLINGFRAME._serialized_end = 1418 + _DATATRANSFERSERVICE._serialized_start = 1502 + _DATATRANSFERSERVICE._serialized_end = 2004 + _ROUTESERVICE._serialized_start = 2006 + _ROUTESERVICE._serialized_end = 2122 +# @@protoc_insertion_point(module_scope) diff --git a/python/fate_flow/proto/rollsite/proxy_pb2_grpc.py b/python/fate_flow/proto/rollsite/proxy_pb2_grpc.py new file mode 100644 index 000000000..9a6dbbad9 --- /dev/null +++ b/python/fate_flow/proto/rollsite/proxy_pb2_grpc.py @@ -0,0 +1,303 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import basic_meta_pb2 as basic__meta__pb2 +import grpc +import proxy_pb2 as proxy__pb2 + + +class DataTransferServiceStub(object): + """data transfer service""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.push = channel.stream_unary( + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/push", + request_serializer=proxy__pb2.Packet.SerializeToString, + response_deserializer=proxy__pb2.Metadata.FromString, + ) + self.pull = channel.unary_stream( + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/pull", + request_serializer=proxy__pb2.Metadata.SerializeToString, + response_deserializer=proxy__pb2.Packet.FromString, + ) + self.unaryCall = channel.unary_unary( + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/unaryCall", + request_serializer=proxy__pb2.Packet.SerializeToString, + response_deserializer=proxy__pb2.Packet.FromString, + ) + self.polling = channel.stream_stream( + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/polling", + request_serializer=proxy__pb2.PollingFrame.SerializeToString, + response_deserializer=proxy__pb2.PollingFrame.FromString, + ) + + +class DataTransferServiceServicer(object): + """data transfer service""" + + def push(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def pull(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def unaryCall(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def polling(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + +def add_DataTransferServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + "push": grpc.stream_unary_rpc_method_handler( + servicer.push, + request_deserializer=proxy__pb2.Packet.FromString, + response_serializer=proxy__pb2.Metadata.SerializeToString, + ), + "pull": grpc.unary_stream_rpc_method_handler( + servicer.pull, + request_deserializer=proxy__pb2.Metadata.FromString, + response_serializer=proxy__pb2.Packet.SerializeToString, + ), + "unaryCall": grpc.unary_unary_rpc_method_handler( + servicer.unaryCall, + request_deserializer=proxy__pb2.Packet.FromString, + response_serializer=proxy__pb2.Packet.SerializeToString, + ), + "polling": grpc.stream_stream_rpc_method_handler( + servicer.polling, + request_deserializer=proxy__pb2.PollingFrame.FromString, + response_serializer=proxy__pb2.PollingFrame.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + "com.webank.ai.eggroll.api.networking.proxy.DataTransferService", + rpc_method_handlers, + ) + server.add_generic_rpc_handlers((generic_handler,)) + + +# This class is part of an EXPERIMENTAL API. +class DataTransferService(object): + """data transfer service""" + + @staticmethod + def push( + request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.stream_unary( + request_iterator, + target, + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/push", + proxy__pb2.Packet.SerializeToString, + proxy__pb2.Metadata.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def pull( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_stream( + request, + target, + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/pull", + proxy__pb2.Metadata.SerializeToString, + proxy__pb2.Packet.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def unaryCall( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/unaryCall", + proxy__pb2.Packet.SerializeToString, + proxy__pb2.Packet.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def polling( + request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.stream_stream( + request_iterator, + target, + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/polling", + proxy__pb2.PollingFrame.SerializeToString, + proxy__pb2.PollingFrame.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + +class RouteServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.query = channel.unary_unary( + "/com.webank.ai.eggroll.api.networking.proxy.RouteService/query", + request_serializer=proxy__pb2.Topic.SerializeToString, + response_deserializer=basic__meta__pb2.Endpoint.FromString, + ) + + +class RouteServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def query(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + +def add_RouteServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + "query": grpc.unary_unary_rpc_method_handler( + servicer.query, + request_deserializer=proxy__pb2.Topic.FromString, + response_serializer=basic__meta__pb2.Endpoint.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + "com.webank.ai.eggroll.api.networking.proxy.RouteService", rpc_method_handlers + ) + server.add_generic_rpc_handlers((generic_handler,)) + + +# This class is part of an EXPERIMENTAL API. +class RouteService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def query( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/com.webank.ai.eggroll.api.networking.proxy.RouteService/query", + proxy__pb2.Topic.SerializeToString, + basic__meta__pb2.Endpoint.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) diff --git a/python/fate_flow/protobuf/proto/pipeline.proto b/python/fate_flow/protobuf/proto/pipeline.proto deleted file mode 100644 index a08266e29..000000000 --- a/python/fate_flow/protobuf/proto/pipeline.proto +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2019 The FATE Authors. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto3"; - -package com.webank.ai.fate.core.mlmodel.buffer; -option java_outer_classname = "PipelineProto"; - - -message Pipeline { - bytes inference_dsl = 1; - bytes train_dsl = 2; - bytes train_runtime_conf = 3; - string fate_version = 4; - string model_id = 5; - string model_version = 6; - bool parent = 7; - int32 loaded_times = 8; - bytes roles = 9; - int32 work_mode = 10; - string initiator_role = 11; - int32 initiator_party_id = 12; - bytes runtime_conf_on_party = 13; - bytes parent_info = 14; -} diff --git a/python/fate_flow/protobuf/python/pipeline_pb2.py b/python/fate_flow/protobuf/python/pipeline_pb2.py deleted file mode 100644 index 9d721fa38..000000000 --- a/python/fate_flow/protobuf/python/pipeline_pb2.py +++ /dev/null @@ -1,162 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: pipeline.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='pipeline.proto', - package='com.webank.ai.fate.core.mlmodel.buffer', - syntax='proto3', - serialized_options=_b('B\rPipelineProto'), - serialized_pb=_b('\n\x0epipeline.proto\x12&com.webank.ai.fate.core.mlmodel.buffer\"\xbf\x02\n\x08Pipeline\x12\x15\n\rinference_dsl\x18\x01 \x01(\x0c\x12\x11\n\ttrain_dsl\x18\x02 \x01(\x0c\x12\x1a\n\x12train_runtime_conf\x18\x03 \x01(\x0c\x12\x14\n\x0c\x66\x61te_version\x18\x04 \x01(\t\x12\x10\n\x08model_id\x18\x05 \x01(\t\x12\x15\n\rmodel_version\x18\x06 \x01(\t\x12\x0e\n\x06parent\x18\x07 \x01(\x08\x12\x14\n\x0cloaded_times\x18\x08 \x01(\x05\x12\r\n\x05roles\x18\t \x01(\x0c\x12\x11\n\twork_mode\x18\n \x01(\x05\x12\x16\n\x0einitiator_role\x18\x0b \x01(\t\x12\x1a\n\x12initiator_party_id\x18\x0c \x01(\x05\x12\x1d\n\x15runtime_conf_on_party\x18\r \x01(\x0c\x12\x13\n\x0bparent_info\x18\x0e \x01(\x0c\x42\x0f\x42\rPipelineProtob\x06proto3') -) - - - - -_PIPELINE = _descriptor.Descriptor( - name='Pipeline', - full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='inference_dsl', full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline.inference_dsl', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='train_dsl', full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline.train_dsl', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='train_runtime_conf', full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline.train_runtime_conf', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='fate_version', full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline.fate_version', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='model_id', full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline.model_id', index=4, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='model_version', full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline.model_version', index=5, - number=6, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='parent', full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline.parent', index=6, - number=7, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='loaded_times', full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline.loaded_times', index=7, - number=8, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='roles', full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline.roles', index=8, - number=9, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='work_mode', full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline.work_mode', index=9, - number=10, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='initiator_role', full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline.initiator_role', index=10, - number=11, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='initiator_party_id', full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline.initiator_party_id', index=11, - number=12, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='runtime_conf_on_party', full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline.runtime_conf_on_party', index=12, - number=13, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='parent_info', full_name='com.webank.ai.fate.core.mlmodel.buffer.Pipeline.parent_info', index=13, - number=14, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=59, - serialized_end=378, -) - -DESCRIPTOR.message_types_by_name['Pipeline'] = _PIPELINE -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Pipeline = _reflection.GeneratedProtocolMessageType('Pipeline', (_message.Message,), { - 'DESCRIPTOR' : _PIPELINE, - '__module__' : 'pipeline_pb2' - # @@protoc_insertion_point(class_scope:com.webank.ai.fate.core.mlmodel.buffer.Pipeline) - }) -_sym_db.RegisterMessage(Pipeline) - - -DESCRIPTOR._options = None -# @@protoc_insertion_point(module_scope) diff --git a/python/fate_flow/runtime/__init__.py b/python/fate_flow/runtime/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/fate_flow/runtime/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/fate_flow/entity/_component_provider.py b/python/fate_flow/runtime/component_provider.py similarity index 54% rename from python/fate_flow/entity/_component_provider.py rename to python/fate_flow/runtime/component_provider.py index f8016b7e6..0db4932f5 100644 --- a/python/fate_flow/entity/_component_provider.py +++ b/python/fate_flow/runtime/component_provider.py @@ -13,51 +13,54 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import os +from typing import Union -from fate_flow.entity.types import ComponentProviderName from fate_flow.entity import BaseEntity +from fate_flow.entity.spec.flow import ProviderSpec, LocalProviderSpec, DockerProviderSpec, K8sProviderSpec class ComponentProvider(BaseEntity): - def __init__(self, name: str, version: str, path: str, class_path: dict, _python_env="",**kwargs): - if not ComponentProviderName.valid(name): - raise ValueError(f"not support {name} provider") - self._name = name - self._version = version - self._path = os.path.abspath(path) - self._class_path = class_path - self._python_env = _python_env - self._env = {} + def __init__(self, provider_info: ProviderSpec): + self._name = provider_info.name + self._device = provider_info.device + self._version = provider_info.version + self._metadata = provider_info.metadata + self._python_path = "" + self._python_env = "" self.init_env() def init_env(self): - self._env["PYTHONPATH"] = os.path.dirname(self._path) - self._env["PYTHON_ENV"] = self.python_env + if isinstance(self._metadata, LocalProviderSpec): + self._python_path = self._metadata.path + self._python_env = self._metadata.venv @property def name(self): return self._name @property - def version(self): - return self._version + def device(self): + return self._device @property - def path(self): - return self._path + def version(self): + return self._version @property - def class_path(self): - return self._class_path + def metadata(self) -> Union[LocalProviderSpec, DockerProviderSpec, K8sProviderSpec]: + return self._metadata @property - def env(self): - return self._env + def python_path(self): + return self._python_path @property def python_env(self): return self._python_env + @property + def provider_name(self): + return f"{self.name}:{self.version}@{self.device}" + def __eq__(self, other): return self.name == other.name and self.version == other.version diff --git a/python/fate_flow/runtime/env.py b/python/fate_flow/runtime/env.py new file mode 100644 index 000000000..d66c87751 --- /dev/null +++ b/python/fate_flow/runtime/env.py @@ -0,0 +1,10 @@ +import sys +import fate_flow + + +def is_in_virtualenv(): + try: + module_path = fate_flow.__file__ + return sys.prefix in module_path + except ImportError: + return False diff --git a/python/fate_flow/db/job_default_config.py b/python/fate_flow/runtime/job_default_config.py similarity index 57% rename from python/fate_flow/db/job_default_config.py rename to python/fate_flow/runtime/job_default_config.py index 6a98a9b5a..732f92e11 100644 --- a/python/fate_flow/db/job_default_config.py +++ b/python/fate_flow/runtime/job_default_config.py @@ -13,44 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from fate_arch.common import file_utils -from fate_flow.settings import FATE_FLOW_JOB_DEFAULT_CONFIG_PATH, stat_logger +from fate_flow.runtime.system_settings import FATE_FLOW_JOB_DEFAULT_CONFIG_PATH from .reload_config_base import ReloadConfigBase +from ..utils import file_utils +from ..utils.log import getLogger +stat_logger = getLogger() -class JobDefaultConfig(ReloadConfigBase): - # component provider - default_component_provider_path = None - - # Resource - total_cores_overweight_percent = None - total_memory_overweight_percent = None - task_parallelism = None - task_cores = None - task_memory = None - max_cores_percent_per_job = None - # scheduling +class JobDefaultConfig(ReloadConfigBase): + job_cores = None + computing_partitions = None + task_run = None remote_request_timeout = None federated_command_trys = None job_timeout = None - end_status_job_scheduling_time_limit = None - end_status_job_scheduling_updates = None auto_retries = None - auto_retry_delay = None - federated_status_collect_type = None - detect_connect_max_retry_count = None - detect_connect_long_retry_count = None - - # upload - upload_block_max_bytes = None # bytes - - # component output - output_data_summary_count_limit = None + sync_type = None - task_world_size = None - resource_waiting_timeout = None - task_process_classpath = None + task_logger = None + task_device = None + task_timeout = None @classmethod def load(cls): @@ -64,4 +47,4 @@ def load(cls): else: stat_logger.warning(f"job default config not supported {k}") - return cls.get_all() \ No newline at end of file + return cls.get_all() diff --git a/python/fate_flow/db/reload_config_base.py b/python/fate_flow/runtime/reload_config_base.py similarity index 100% rename from python/fate_flow/db/reload_config_base.py rename to python/fate_flow/runtime/reload_config_base.py diff --git a/python/fate_flow/db/runtime_config.py b/python/fate_flow/runtime/runtime_config.py similarity index 72% rename from python/fate_flow/db/runtime_config.py rename to python/fate_flow/runtime/runtime_config.py index b11379bd7..cfee7cb70 100644 --- a/python/fate_flow/db/runtime_config.py +++ b/python/fate_flow/runtime/runtime_config.py @@ -13,32 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from fate_arch.common.versions import get_versions from fate_flow.entity.types import ProcessRole -from fate_flow.entity import ComponentProvider -from .reload_config_base import ReloadConfigBase +from fate_flow.runtime.reload_config_base import ReloadConfigBase +from fate_flow.utils.version import get_versions -class RuntimeConfig(ReloadConfigBase): - DEBUG = None - WORK_MODE = None - COMPUTING_ENGINE = None - FEDERATION_ENGINE = None - FEDERATED_MODE = None - JOB_QUEUE = None - USE_LOCAL_DATABASE = False +class RuntimeConfig(ReloadConfigBase): HTTP_PORT = None JOB_SERVER_HOST = None - JOB_SERVER_VIP = None - IS_SERVER = False PROCESS_ROLE = None - ENV = dict() - COMPONENT_PROVIDER: ComponentProvider = None + SCHEDULE_CLIENT = None + CLIENT_ROLE = list() SERVICE_DB = None - LOAD_COMPONENT_REGISTRY = False - LOAD_CONFIG_MANAGER = False SESSION_LIST = [] + ENV = dict() @classmethod def init_config(cls, **kwargs): @@ -71,8 +60,14 @@ def set_process_role(cls, process_role: ProcessRole): cls.PROCESS_ROLE = process_role @classmethod - def set_component_provider(cls, component_provider: ComponentProvider): - cls.COMPONENT_PROVIDER = component_provider + def set_schedule_client(cls, schedule_client): + cls.SCHEDULE_CLIENT = schedule_client + + @classmethod + def set_client_roles(cls, *roles): + for role in roles: + if role not in cls.CLIENT_ROLE: + cls.CLIENT_ROLE.append(role) @classmethod def set_service_db(cls, service_db): diff --git a/python/fate_flow/runtime/system_settings.py b/python/fate_flow/runtime/system_settings.py new file mode 100644 index 000000000..245c864e6 --- /dev/null +++ b/python/fate_flow/runtime/system_settings.py @@ -0,0 +1,151 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +from grpc._cython import cygrpc + +from fate_flow.entity.types import ComputingEngine +from fate_flow.runtime.env import is_in_virtualenv +from fate_flow.utils import engine_utils, file_utils +from fate_flow.utils.conf_utils import get_base_config +from fate_flow.utils.file_utils import get_fate_flow_directory, get_fate_python_path + +from fate_flow.settings import * + +# Server +API_VERSION = "v2" +FATE_FLOW_SERVICE_NAME = "fateflow" +SERVER_MODULE = "fate_flow_server.py" +CASBIN_TABLE_NAME = "fate_casbin" +PERMISSION_TABLE_NAME = "permission_casbin" +PERMISSION_MANAGER_PAGE = "permission" +APP_MANAGER_PAGE = "app" + +ADMIN_PAGE = [PERMISSION_MANAGER_PAGE, APP_MANAGER_PAGE] +FATE_FLOW_CONF_PATH = os.path.join(get_fate_flow_directory(), "conf") + +FATE_FLOW_JOB_DEFAULT_CONFIG_PATH = os.path.join(FATE_FLOW_CONF_PATH, "job_default_config.yaml") + +SUBPROCESS_STD_LOG_NAME = "std.log" + +HOST = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("host", "127.0.0.1") +HTTP_PORT = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("http_port") +GRPC_PORT = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("grpc_port") + +NGINX_HOST = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("nginx", {}).get("host") or HOST +NGINX_HTTP_PORT = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("nginx", {}).get("http_port") or HTTP_PORT +RANDOM_INSTANCE_ID = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("random_instance_id", False) + +PROTOCOL = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("protocol", "http") + +PROXY_NAME = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("proxy_name") +PROXY_PROTOCOL = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("protocol", "http") +PROXY = get_base_config("federation") +STORAGE = get_base_config("storage") +ENGINES = engine_utils.get_engines() +IS_STANDALONE = engine_utils.is_standalone() +WORKER = get_base_config("worker", {}) +DEFAULT_PROVIDER = get_base_config("default_provider", {}) +CASBIN_MODEL_CONF = os.path.join(FATE_FLOW_CONF_PATH, "casbin_model.conf") +PERMISSION_CASBIN_MODEL_CONF = os.path.join(FATE_FLOW_CONF_PATH, "permission_casbin_model.conf") +SERVICE_CONF_NAME = "service_conf.yaml" + +DATABASE = get_base_config("database", {}) + +IGNORE_RESOURCE_ROLES = {"arbiter"} + +SUPPORT_IGNORE_RESOURCE_ENGINES = { + ComputingEngine.EGGROLL, ComputingEngine.STANDALONE +} +DEFAULT_FATE_PROVIDER_PATH = (DEFAULT_FATE_DIR or get_fate_python_path()) if not is_in_virtualenv() else "" +HEADERS = { + "Content-Type": "application/json", + "Connection": "close", + "service": FATE_FLOW_SERVICE_NAME +} + +BASE_URI = f"{PROTOCOL}://{HOST}:{HTTP_PORT}/{API_VERSION}" + +HOOK_MODULE = get_base_config("hook_module") +# computing +COMPUTING_CONF = get_base_config("computing", {}) + +# authentication +AUTHENTICATION_CONF = get_base_config("authentication", {}) +# client +CLIENT_AUTHENTICATION = AUTHENTICATION_CONF.get("client", False) +# site +SITE_AUTHENTICATION = AUTHENTICATION_CONF.get("site", False) +# permission +PERMISSION_SWITCH = AUTHENTICATION_CONF.get("permission", False) + +ENCRYPT_CONF = get_base_config("encrypt") + +PARTY_ID = get_base_config("party_id", "") +LOCAL_PARTY_ID = "0" + +MODEL_STORE = get_base_config("model_store") + +GRPC_OPTIONS = [ + (cygrpc.ChannelArgKey.max_send_message_length, -1), + (cygrpc.ChannelArgKey.max_receive_message_length, -1), +] + +LOG_DIR = LOG_DIR or get_fate_flow_directory("logs") +JOB_DIR = JOB_DIR or get_fate_flow_directory("jobs") +MODEL_STORE_PATH = MODEL_DIR or os.path.join(get_fate_flow_directory(), "model") +STANDALONE_DATA_HOME = DATA_DIR or os.path.join(file_utils.get_fate_flow_directory(), "data") +LOCALFS_DATA_HOME = DATA_DIR or os.path.join(file_utils.get_fate_flow_directory(), "localfs") +TEMP_DIR = get_fate_flow_directory("temps") +LOG_LEVEL = int(os.environ.get("LOG_LEVEL") or get_base_config("log_level", 10)) +LOG_SHARE = False +FATE_FLOW_LOG_DIR = os.path.join(LOG_DIR, "fate_flow") +WORKERS_DIR = os.path.join(LOG_DIR, "workers") + +SQLITE_FILE_DIR = SQLITE_FILE_DIR or get_fate_flow_directory() +SQLITE_PATH = os.path.join(SQLITE_FILE_DIR, SQLITE_FILE_NAME) + +GRPC_SERVER_MAX_WORKERS = GRPC_SERVER_MAX_WORKERS or (os.cpu_count() or 1) * 20 + +VERSION_FILE_PATH = os.path.join(get_fate_flow_directory(), "fateflow.env") +FATE_FLOW_PROVIDER_PATH = get_fate_flow_directory("python") + +# Registry +FATE_FLOW_MODEL_TRANSFER_ENDPOINT = "/v1/model/transfer" +ZOOKEEPER = get_base_config("zookeeper", {}) +ZOOKEEPER_REGISTRY = { + # server + 'flow-server': "/FATE-COMPONENTS/fate-flow", + # model service + 'fateflow': "/FATE-SERVICES/flow/online/transfer/providers", + 'servings': "/FATE-SERVICES/serving/online/publishLoad/providers", +} +USE_REGISTRY = get_base_config("use_registry") + +DEEPSPEED_LOGS_DIR_PLACEHOLDER = "EGGROLL_DEEPSPEED_LOGS_DIR" +DEEPSPEED_MODEL_DIR_PLACEHOLDER = "EGGROLL_DEEPSPEED_MODEL_DIR" +DEEPSPEED_RESULT_PLACEHOLDER = "EGGROLL_DEEPSPEED_RESULT_DIR" + +REQUEST_TRY_TIMES = 3 +REQUEST_WAIT_SEC = 2 +REQUEST_MAX_WAIT_SEC = 300 +SESSION_VALID_PERIOD = 7 * 24 * 60 * 60 * 1000 + +DEFAULT_OUTPUT_DATA_PARTITIONS = 16 + +# hub module settings +# define： xxx.class_name +DEFAULT_COMPONENTS_WRAPS_MODULE = "fate_flow.hub.components_wraps.fate.FlowWraps" diff --git a/python/fate_flow/scheduler/__init__.py b/python/fate_flow/scheduler/__init__.py index 25e85b6b4..a6caef259 100644 --- a/python/fate_flow/scheduler/__init__.py +++ b/python/fate_flow/scheduler/__init__.py @@ -12,26 +12,29 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -from fate_flow.entity import RetCode -from fate_flow.entity.run_status import FederatedSchedulingStatusCode +from fate_flow.scheduler.scheduler import SchedulerABC +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.runtime.system_settings import HOST, HTTP_PORT, PROXY_PROTOCOL, API_VERSION, HTTP_REQUEST_TIMEOUT +from fate_flow.utils.api_utils import get_federated_proxy_address, generate_headers +from ofx.api.client import FlowSchedulerApi + + +def init_scheduler(): + remote_host, remote_port, remote_protocol, grpc_channel = get_federated_proxy_address() + + protocol = remote_protocol if remote_protocol else PROXY_PROTOCOL + # schedule client + RuntimeConfig.set_schedule_client( + FlowSchedulerApi( + host=HOST, + port=HTTP_PORT, + api_version=API_VERSION, + timeout=HTTP_REQUEST_TIMEOUT, + remote_protocol=protocol, + remote_host=remote_host, + remote_port=remote_port, + grpc_channel=grpc_channel, + callback=generate_headers) + ) -class SchedulerBase(): - @classmethod - def return_federated_response(cls, federated_response): - retcode_set = set() - for dest_role in federated_response.keys(): - for party_id in federated_response[dest_role].keys(): - retcode_set.add(federated_response[dest_role][party_id]["retcode"]) - if len(retcode_set) == 1 and RetCode.SUCCESS in retcode_set: - federated_scheduling_status_code = FederatedSchedulingStatusCode.SUCCESS - elif RetCode.EXCEPTION_ERROR in retcode_set: - federated_scheduling_status_code = FederatedSchedulingStatusCode.ERROR - elif RetCode.NOT_EFFECTIVE in retcode_set: - federated_scheduling_status_code = FederatedSchedulingStatusCode.NOT_EFFECTIVE - elif RetCode.SUCCESS in retcode_set: - federated_scheduling_status_code = FederatedSchedulingStatusCode.PARTIAL - else: - federated_scheduling_status_code = FederatedSchedulingStatusCode.FAILED - return federated_scheduling_status_code, federated_response diff --git a/python/fate_flow/scheduler/cluster_scheduler.py b/python/fate_flow/scheduler/cluster_scheduler.py deleted file mode 100644 index c4a428291..000000000 --- a/python/fate_flow/scheduler/cluster_scheduler.py +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright 2022 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from concurrent.futures import ThreadPoolExecutor, as_completed -from unittest import result - -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.entity import RetCode -from fate_flow.scheduler import SchedulerBase -from fate_flow.utils.api_utils import cluster_api -from fate_flow.utils.log_utils import failed_log, schedule_logger, start_log - - -class ClusterScheduler(SchedulerBase): - - @classmethod - def cluster_command(cls, endpoint, json_body): - log_msg = f'sending {endpoint} cluster federated command' - schedule_logger().info(start_log(msg=log_msg)) - - instance_list = RuntimeConfig.SERVICE_DB.get_servers() - result = {} - - with ThreadPoolExecutor(max_workers=len(instance_list)) as executor: - futures = { - executor.submit( - cluster_api, - method='POST', - host=instance.host, - port=instance.http_port, - endpoint=endpoint, - json_body=json_body, - ): instance_id - for instance_id, instance in instance_list.items() - } - - for future in as_completed(futures): - instance = instance_list[futures[future]] - - try: - response = future.result() - except Exception as e: - schedule_logger().exception(e) - - response = { - 'retcode': RetCode.FEDERATED_ERROR, - 'retmsg': f'Federated schedule error: {instance.instance_id}\n{e}', - } - else: - if response['retcode'] != RetCode.SUCCESS: - schedule_logger().error(failed_log(msg=log_msg, detail=response)) - - result[instance.instance_id] = response - - return result diff --git a/python/fate_flow/scheduler/dag_scheduler.py b/python/fate_flow/scheduler/dag_scheduler.py deleted file mode 100644 index 2947d3713..000000000 --- a/python/fate_flow/scheduler/dag_scheduler.py +++ /dev/null @@ -1,614 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import typing -from copy import deepcopy - -from fate_arch.common import FederatedMode -from fate_arch.common.base_utils import current_timestamp, json_dumps, json_loads - -from fate_flow.controller.job_controller import JobController -from fate_flow.db.db_models import DB, Job, Task -from fate_flow.db.job_default_config import JobDefaultConfig -from fate_flow.entity import JobConfigurationBase, RetCode -from fate_flow.entity.run_status import ( - EndStatus, FederatedSchedulingStatusCode, InterruptStatus, JobInheritanceStatus, - JobStatus, SchedulingStatusCode, StatusSet, TaskStatus, -) -from fate_flow.entity.types import ResourceOperation -from fate_flow.model.sync_model import SyncModel -from fate_flow.operation.job_saver import JobSaver -from fate_flow.operation.job_tracker import Tracker -from fate_flow.scheduler.federated_scheduler import FederatedScheduler -from fate_flow.scheduler.task_scheduler import TaskScheduler -from fate_flow.settings import ENABLE_MODEL_STORE -from fate_flow.utils import detect_utils, job_utils, model_utils, schedule_utils -from fate_flow.utils.config_adapter import JobRuntimeConfigAdapter -from fate_flow.utils.cron import Cron -from fate_flow.utils.log_utils import exception_to_trace_string, schedule_logger - - -class DAGScheduler(Cron): - @classmethod - def submit(cls, submit_job_conf: JobConfigurationBase, job_id: str = None): - if not job_id: - job_id = job_utils.generate_job_id() - submit_result = { - "job_id": job_id - } - schedule_logger(job_id).info(f"submit job, body {submit_job_conf.to_dict()}") - try: - dsl = submit_job_conf.dsl - runtime_conf = deepcopy(submit_job_conf.runtime_conf) - job_utils.check_job_conf(runtime_conf, dsl) - job_initiator = runtime_conf["initiator"] - conf_adapter = JobRuntimeConfigAdapter(runtime_conf) - common_job_parameters = conf_adapter.get_common_parameters() - common_job_parameters.roles = runtime_conf["role"] - common_job_parameters.role_parameters = runtime_conf.get("job_parameters", {}).get("role", {}) - - if common_job_parameters.job_type != "predict": - # generate job model info - conf_version = schedule_utils.get_conf_version(runtime_conf) - if conf_version != 2: - raise Exception("only the v2 version runtime conf is supported") - common_job_parameters.model_id = model_utils.gen_model_id(runtime_conf["role"]) - common_job_parameters.model_version = job_id - train_runtime_conf = {} - else: - # check predict job parameters - detect_utils.check_config(common_job_parameters.to_dict(), ["model_id", "model_version"]) - - # get inference dsl from pipeline model as job dsl - tracker = Tracker(job_id=job_id, role=job_initiator["role"], party_id=job_initiator["party_id"], - model_id=common_job_parameters.model_id, model_version=common_job_parameters.model_version) - - if ENABLE_MODEL_STORE: - sync_model = SyncModel( - role=tracker.role, party_id=tracker.party_id, - model_id=tracker.model_id, model_version=tracker.model_version, - ) - if sync_model.remote_exists(): - sync_model.download(True) - - if not model_utils.check_if_deployed( - role=tracker.role, party_id=tracker.party_id, - model_id=tracker.model_id, model_version=tracker.model_version, - ): - raise Exception(f"model has not been deployed yet") - - pipeline_model = tracker.pipelined_model.read_pipeline_model() - train_runtime_conf = json_loads(pipeline_model.train_runtime_conf) - dsl = json_loads(pipeline_model.inference_dsl) - - job = Job() - job.f_job_id = job_id - job.f_dsl = dsl - job.f_train_runtime_conf = train_runtime_conf - job.f_roles = runtime_conf["role"] - job.f_initiator_role = job_initiator["role"] - job.f_initiator_party_id = job_initiator["party_id"] - job.f_role = job_initiator["role"] - job.f_party_id = job_initiator["party_id"] - - path_dict = job_utils.save_job_conf(job_id=job_id, - role=job.f_initiator_role, - party_id=job.f_initiator_party_id, - dsl=dsl, - runtime_conf=runtime_conf, - runtime_conf_on_party={}, - train_runtime_conf=train_runtime_conf, - pipeline_dsl=None) - - if job.f_initiator_party_id not in runtime_conf["role"][job.f_initiator_role]: - msg = f"initiator party id {job.f_initiator_party_id} not in roles {runtime_conf['role']}" - schedule_logger(job_id).info(msg) - raise Exception(msg) - - # create common parameters on initiator - JobController.create_common_job_parameters(job_id=job.f_job_id, initiator_role=job.f_initiator_role, common_job_parameters=common_job_parameters) - job.f_runtime_conf = conf_adapter.update_common_parameters(common_parameters=common_job_parameters) - dsl_parser = schedule_utils.get_job_dsl_parser(dsl=job.f_dsl, - runtime_conf=job.f_runtime_conf, - train_runtime_conf=job.f_train_runtime_conf) - - # initiator runtime conf as template - job.f_runtime_conf_on_party = job.f_runtime_conf.copy() - job.f_runtime_conf_on_party["job_parameters"] = common_job_parameters.to_dict() - - # inherit job - job.f_inheritance_info = common_job_parameters.inheritance_info - job.f_inheritance_status = JobInheritanceStatus.WAITING if common_job_parameters.inheritance_info else JobInheritanceStatus.PASS - if job.f_inheritance_info: - inheritance_jobs = JobSaver.query_job(job_id=job.f_inheritance_info.get("job_id"), role=job_initiator["role"], party_id=job_initiator["party_id"]) - inheritance_tasks = JobSaver.query_task(job_id=job.f_inheritance_info.get("job_id"), role=job_initiator["role"], party_id=job_initiator["party_id"], only_latest=True) - job_utils.check_job_inheritance_parameters(job, inheritance_jobs, inheritance_tasks) - - status_code, response = FederatedScheduler.create_job(job=job) - if status_code != FederatedSchedulingStatusCode.SUCCESS: - job.f_status = JobStatus.FAILED - job.f_tag = "submit_failed" - FederatedScheduler.sync_job_status(job=job) - raise Exception("create job failed", response) - else: - need_run_components = {} - for role in response: - need_run_components[role] = {} - for party, res in response[role].items(): - need_run_components[role][party] = [name for name, value in response[role][party]["data"]["components"].items() if value["need_run"] is True] - if common_job_parameters.federated_mode == FederatedMode.MULTIPLE: - # create the task holder in db to record information of all participants in the initiator for scheduling - for role, party_ids in job.f_roles.items(): - for party_id in party_ids: - if role == job.f_initiator_role and party_id == job.f_initiator_party_id: - continue - if not need_run_components[role][party_id]: - continue - JobController.initialize_tasks(job_id=job_id, - role=role, - party_id=party_id, - run_on_this_party=False, - initiator_role=job.f_initiator_role, - initiator_party_id=job.f_initiator_party_id, - job_parameters=common_job_parameters, - dsl_parser=dsl_parser, - components=need_run_components[role][party_id], - runtime_conf=runtime_conf, - is_scheduler=True) - job.f_status = JobStatus.WAITING - status_code, response = FederatedScheduler.sync_job_status(job=job) - if status_code != FederatedSchedulingStatusCode.SUCCESS: - raise Exception(f"set job to waiting status failed: {response}") - - schedule_logger(job_id).info(f"submit job successfully, job id is {job.f_job_id}, model id is {common_job_parameters.model_id}") - logs_directory = job_utils.get_job_log_directory(job_id) - result = { - "code": RetCode.SUCCESS, - "message": "success", - "model_info": {"model_id": common_job_parameters.model_id, "model_version": common_job_parameters.model_version}, - "logs_directory": logs_directory, - "board_url": job_utils.get_board_url(job_id, job_initiator["role"], job_initiator["party_id"]) - } - warn_parameter = JobRuntimeConfigAdapter(submit_job_conf.runtime_conf).check_removed_parameter() - if warn_parameter: - result["message"] = f"[WARN]{warn_parameter} is removed,it does not take effect!" - submit_result.update(result) - submit_result.update(path_dict) - except Exception as e: - submit_result["code"] = RetCode.OPERATING_ERROR - submit_result["message"] = exception_to_trace_string(e) - schedule_logger(job_id).exception(e) - return submit_result - - @classmethod - def update_parameters(cls, job, job_parameters, component_parameters): - updated_job_parameters, updated_component_parameters, updated_components = JobController.gen_updated_parameters(job_id=job.f_job_id, - initiator_role=job.f_initiator_role, - initiator_party_id=job.f_initiator_party_id, - input_job_parameters=job_parameters, - input_component_parameters=component_parameters) - schedule_logger(job.f_job_id).info(f"components {updated_components} parameters has been updated") - updated_parameters = { - "job_parameters": updated_job_parameters, - "component_parameters": updated_component_parameters, - "components": updated_components - } - status_code, response = FederatedScheduler.update_parameter(job, updated_parameters=updated_parameters) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - return RetCode.SUCCESS, updated_parameters - else: - return RetCode.OPERATING_ERROR, response - - def run_do(self): - schedule_logger().info("start schedule waiting jobs") - jobs = JobSaver.query_job(is_initiator=True, status=JobStatus.WAITING, order_by="create_time", reverse=False) - schedule_logger().info(f"have {len(jobs)} waiting jobs") - if len(jobs): - # FIFO - job = jobs[0] - schedule_logger().info(f"schedule waiting job {job.f_job_id}") - try: - self.schedule_waiting_jobs(job=job, lock=True) - except Exception as e: - schedule_logger(job.f_job_id).exception(e) - schedule_logger(job.f_job_id).error("schedule waiting job failed") - schedule_logger().info("schedule waiting jobs finished") - - schedule_logger().info("start schedule running jobs") - jobs = JobSaver.query_job(is_initiator=True, status=JobStatus.RUNNING, order_by="create_time", reverse=False) - schedule_logger().info(f"have {len(jobs)} running jobs") - for job in jobs: - schedule_logger().info(f"schedule running job {job.f_job_id}") - try: - self.schedule_running_job(job=job, lock=True) - except Exception as e: - schedule_logger(job.f_job_id).exception(e) - schedule_logger(job.f_job_id).error("schedule job failed") - schedule_logger().info("schedule running jobs finished") - - # some ready job exit before start - schedule_logger().info("start schedule ready jobs") - jobs = JobSaver.query_job(is_initiator=True, ready_signal=True, order_by="create_time", reverse=False) - schedule_logger().info(f"have {len(jobs)} ready jobs") - for job in jobs: - schedule_logger().info(f"schedule ready job {job.f_job_id}") - try: - self.schedule_ready_job(job=job) - except Exception as e: - schedule_logger(job.f_job_id).exception(e) - schedule_logger(job.f_job_id).error(f"schedule ready job failed:\n{e}") - schedule_logger().info("schedule ready jobs finished") - - schedule_logger().info("start schedule rerun jobs") - jobs = JobSaver.query_job(is_initiator=True, rerun_signal=True, order_by="create_time", reverse=False) - schedule_logger().info(f"have {len(jobs)} rerun jobs") - for job in jobs: - schedule_logger(job.f_job_id).info(f"schedule rerun job {job.f_job_id}") - try: - self.schedule_rerun_job(job=job, lock=True) - except Exception as e: - schedule_logger(job.f_job_id).exception(e) - schedule_logger(job.f_job_id).error("schedule job failed") - schedule_logger().info("schedule rerun jobs finished") - - schedule_logger().info("start schedule end status jobs to update status") - jobs = JobSaver.query_job(is_initiator=True, status=set(EndStatus.status_list()), end_time=[current_timestamp() - JobDefaultConfig.end_status_job_scheduling_time_limit, current_timestamp()]) - schedule_logger().info(f"have {len(jobs)} end status jobs") - for job in jobs: - schedule_logger().info(f"schedule end status job {job.f_job_id}") - try: - update_status = self.end_scheduling_updates(job_id=job.f_job_id) - if update_status: - schedule_logger(job.f_job_id).info("try update status by scheduling like running job") - else: - schedule_logger(job.f_job_id).info("the number of updates has been exceeded") - continue - self.schedule_running_job(job=job, force_sync_status=True, lock=True) - except Exception as e: - schedule_logger(job.f_job_id).exception(e) - schedule_logger(job.f_job_id).error("schedule job failed") - schedule_logger().info("schedule end status jobs finished") - - @classmethod - @schedule_utils.schedule_lock - def schedule_waiting_jobs(cls, job): - job_id, initiator_role, initiator_party_id, = job.f_job_id, job.f_initiator_role, job.f_initiator_party_id, - if job.f_cancel_signal: - job.f_status = JobStatus.CANCELED - FederatedScheduler.sync_job_status(job=job) - schedule_logger(job_id).info("job have cancel signal") - return - if job.f_inheritance_status != JobInheritanceStatus.PASS: - cls.check_component(job) - schedule_logger(job_id).info("job dependence check") - dependence_status_code, federated_dependence_response = FederatedScheduler.dependence_for_job(job=job) - schedule_logger(job_id).info(f"dependence check: {dependence_status_code}, {federated_dependence_response}") - if dependence_status_code == FederatedSchedulingStatusCode.SUCCESS: - apply_status_code, federated_response = FederatedScheduler.resource_for_job(job=job, operation_type=ResourceOperation.APPLY) - if apply_status_code == FederatedSchedulingStatusCode.SUCCESS: - cls.start_job(job_id=job_id, initiator_role=initiator_role, initiator_party_id=initiator_party_id) - else: - # rollback resource - rollback_party = {} - failed_party = {} - for dest_role in federated_response.keys(): - for dest_party_id in federated_response[dest_role].keys(): - retcode = federated_response[dest_role][dest_party_id]["retcode"] - if retcode == 0: - rollback_party[dest_role] = rollback_party.get(dest_role, []) - rollback_party[dest_role].append(dest_party_id) - else: - failed_party[dest_role] = failed_party.get(dest_role, []) - failed_party[dest_role].append(dest_party_id) - schedule_logger(job_id).info("job apply resource failed on {}, rollback {}".format( - ",".join([",".join([f"{_r}:{_p}" for _p in _ps]) for _r, _ps in failed_party.items()]), - ",".join([",".join([f"{_r}:{_p}" for _p in _ps]) for _r, _ps in rollback_party.items()]), - )) - if rollback_party: - return_status_code, federated_response = FederatedScheduler.resource_for_job(job=job, operation_type=ResourceOperation.RETURN, specific_dest=rollback_party) - if return_status_code != FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job_id).info(f"job return resource failed:\n{federated_response}") - else: - schedule_logger(job_id).info("job no party should be rollback resource") - if apply_status_code == FederatedSchedulingStatusCode.ERROR: - cls.stop_job(job_id=job_id, role=initiator_role, party_id=initiator_party_id, stop_status=JobStatus.FAILED) - schedule_logger(job_id).info("apply resource error, stop job") - else: - retcode_set = set() - for dest_role in federated_dependence_response.keys(): - for party_id in federated_dependence_response[dest_role].keys(): - retcode_set.add(federated_dependence_response[dest_role][party_id]["retcode"]) - if not retcode_set.issubset({RetCode.RUNNING, RetCode.SUCCESS}): - FederatedScheduler.stop_job(job, StatusSet.FAILED) - - @classmethod - def check_component(cls, job, check_type="inheritance"): - schedule_logger(job.f_job_id).info("component check") - dependence_status_code, response = FederatedScheduler.check_component(job=job, check_type=check_type) - schedule_logger(job.f_job_id).info(f"component check response: {response}") - dsl_parser = schedule_utils.get_job_dsl_parser(dsl=job.f_dsl, - runtime_conf=job.f_runtime_conf, - train_runtime_conf=job.f_train_runtime_conf) - component_set = set([cpn.name for cpn in dsl_parser.get_source_connect_sub_graph(job.f_inheritance_info.get("component_list"))]) - for dest_role in response.keys(): - for party_id in response[dest_role].keys(): - component_set = component_set.intersection(set(response[dest_role][party_id].get("data"))) - if component_set != set(job.f_inheritance_info.get("component_list")): - schedule_logger(job.f_job_id).info(f"dsl parser components:{component_set}") - - component_list = [cpn.name for cpn in dsl_parser.get_source_connect_sub_graph(list(component_set))] - schedule_logger(job.f_job_id).info(f"parser result:{component_list}") - command_body = {"inheritance_info": job.f_inheritance_info} - command_body["inheritance_info"].update({"component_list": component_list}) - schedule_logger(job.f_job_id).info(f"start align job info:{command_body}") - status_code, response = FederatedScheduler.align_args(job, command_body=command_body) - schedule_logger(job.f_job_id).info(f"align result:{status_code}, {response}") - schedule_logger(job.f_job_id).info("check success") - - @classmethod - def schedule_ready_job(cls, job): - job_id = job.f_job_id - update_status = schedule_utils.ready_signal(job_id=job_id, set_or_reset=False, ready_timeout_ttl=60 * 1000) - schedule_logger(job_id).info(f"reset job ready signal {update_status}") - - @classmethod - @schedule_utils.schedule_lock - def schedule_rerun_job(cls, job): - if EndStatus.contains(job.f_status): - job.f_status = JobStatus.WAITING - job.f_ready_signal = False - job.f_ready_time = None - job.f_rerun_signal = False - job.f_progress = 0 - job.f_end_time = None - job.f_elapsed = None - schedule_logger(job.f_job_id).info("job has been finished, set waiting to rerun") - status, response = FederatedScheduler.sync_job_status(job=job) - if status == FederatedSchedulingStatusCode.SUCCESS: - schedule_utils.rerun_signal(job_id=job.f_job_id, set_or_reset=False) - FederatedScheduler.sync_job(job=job, update_fields=["ready_signal", "ready_time", "rerun_signal", "progress", "end_time", "elapsed"]) - schedule_logger(job.f_job_id).info("job set waiting to rerun successfully") - else: - schedule_logger(job.f_job_id).info("job set waiting to rerun failed") - else: - schedule_utils.rerun_signal(job_id=job.f_job_id, set_or_reset=False) - cls.schedule_running_job(job) - - @classmethod - def start_job(cls, job_id, initiator_role, initiator_party_id): - schedule_logger(job_id).info(f"try to start job on initiator {initiator_role} {initiator_party_id}") - jobs = JobSaver.query_job(job_id=job_id, role=initiator_role, party_id=initiator_party_id) - if jobs: - job = jobs[0] - FederatedScheduler.start_job(job=job) - schedule_logger(job_id).info(f"start job on initiator {initiator_role} {initiator_party_id}") - else: - schedule_logger(job_id).error(f"can not found job on initiator {initiator_role} {initiator_party_id}") - - @classmethod - @schedule_utils.schedule_lock - def schedule_running_job(cls, job: Job, force_sync_status=False): - schedule_logger(job.f_job_id).info("scheduling running job") - - dsl_parser = schedule_utils.get_job_dsl_parser(dsl=job.f_dsl, - runtime_conf=job.f_runtime_conf_on_party, - train_runtime_conf=job.f_train_runtime_conf) - task_scheduling_status_code, auto_rerun_tasks, tasks = TaskScheduler.schedule(job=job, dsl_parser=dsl_parser, canceled=job.f_cancel_signal) - tasks_status = dict([(task.f_component_name, task.f_status) for task in tasks]) - new_job_status = cls.calculate_job_status(task_scheduling_status_code=task_scheduling_status_code, tasks_status=tasks_status.values()) - if new_job_status == JobStatus.WAITING and job.f_cancel_signal: - new_job_status = JobStatus.CANCELED - total, finished_count = cls.calculate_job_progress(tasks_status=tasks_status) - new_progress = float(finished_count) / total * 100 - schedule_logger(job.f_job_id).info(f"job status is {new_job_status}, calculate by task status list: {tasks_status}") - if new_job_status != job.f_status or new_progress != job.f_progress: - # Make sure to update separately, because these two fields update with anti-weight logic - if int(new_progress) - job.f_progress > 0: - job.f_progress = new_progress - FederatedScheduler.sync_job(job=job, update_fields=["progress"]) - cls.update_job_on_initiator(initiator_job=job, update_fields=["progress"]) - if new_job_status != job.f_status: - job.f_status = new_job_status - if EndStatus.contains(job.f_status): - FederatedScheduler.save_pipelined_model(job=job) - FederatedScheduler.sync_job_status(job=job) - cls.update_job_on_initiator(initiator_job=job, update_fields=["status"]) - if EndStatus.contains(job.f_status): - cls.finish(job=job, end_status=job.f_status) - if auto_rerun_tasks: - schedule_logger(job.f_job_id).info("job have auto rerun tasks") - cls.set_job_rerun(job_id=job.f_job_id, initiator_role=job.f_initiator_role, initiator_party_id=job.f_initiator_party_id, tasks=auto_rerun_tasks, auto=True) - if force_sync_status: - FederatedScheduler.sync_job_status(job=job) - schedule_logger(job.f_job_id).info("finish scheduling running job") - - @classmethod - def set_job_rerun(cls, job_id, initiator_role, initiator_party_id, auto, force=False, - tasks: typing.List[Task] = None, component_name: typing.Union[str, list] = None): - schedule_logger(job_id).info(f"try to rerun job on initiator {initiator_role} {initiator_party_id}") - - jobs = JobSaver.query_job(job_id=job_id, role=initiator_role, party_id=initiator_party_id) - if not jobs: - raise RuntimeError(f"can not found job on initiator {initiator_role} {initiator_party_id}") - job = jobs[0] - - dsl_parser = schedule_utils.get_job_dsl_parser(dsl=job.f_dsl, - runtime_conf=job.f_runtime_conf_on_party, - train_runtime_conf=job.f_train_runtime_conf) - component_name, force = cls.get_rerun_component(component_name, job, dsl_parser, force) - schedule_logger(job_id).info(f"rerun component: {component_name}") - - if tasks: - schedule_logger(job_id).info(f"require {[task.f_component_name for task in tasks]} to rerun") - else: - task_query = { - 'job_id': job_id, - 'role': initiator_role, - 'party_id': initiator_party_id, - } - - if not component_name or component_name == job_utils.PIPELINE_COMPONENT_NAME: - # rerun all tasks - schedule_logger(job_id).info("require all component of pipeline to rerun") - else: - _require_reruns = {component_name} if isinstance(component_name, str) else set(component_name) - _should_reruns = _require_reruns.copy() - for _cpn in _require_reruns: - _components = dsl_parser.get_downstream_dependent_components(_cpn) - for _c in _components: - _should_reruns.add(_c.get_name()) - - schedule_logger(job_id).info(f"require {_require_reruns} to rerun, " - f"and then found {_should_reruns} need be to rerun") - task_query['component_name'] = _should_reruns - - tasks = JobSaver.query_task(**task_query) - - job_can_rerun = any([TaskScheduler.prepare_rerun_task( - job=job, task=task, dsl_parser=dsl_parser, auto=auto, force=force, - ) for task in tasks]) - if not job_can_rerun: - FederatedScheduler.sync_job_status(job=job) - schedule_logger(job_id).info("job no task to rerun") - return False - - schedule_logger(job_id).info("job set rerun signal") - status = schedule_utils.rerun_signal(job_id=job_id, set_or_reset=True) - schedule_logger(job_id).info(f"job set rerun signal {'successfully' if status else 'failed'}") - return True - - @classmethod - def get_rerun_component(cls, component_name, job, dsl_parser, force): - if not component_name or component_name == job_utils.PIPELINE_COMPONENT_NAME: - pass - else: - dependence_status_code, response = FederatedScheduler.check_component(job=job, check_type="rerun") - success_task_list = [task.f_component_name for task in JobSaver.query_task(job_id=job.f_job_id, party_id=job.f_party_id, role=job.f_role, - status=TaskStatus.SUCCESS, only_latest=True)] - component_set = set() - for dest_role in response.keys(): - for party_id in response[dest_role].keys(): - component_set = component_set.union(set(response[dest_role][party_id].get("data"))) - schedule_logger(job.f_job_id).info(f"success task list: {success_task_list}, check failed component list: {list(component_set)}") - need_rerun = [cpn.name for cpn in dsl_parser.get_need_revisit_nodes(success_task_list, list(component_set))] - schedule_logger(job.f_job_id).info(f"need rerun success component: {need_rerun}") - if component_set: - force = True - if isinstance(component_name, str): - component_name = set(need_rerun).union({component_name}) - else: - component_name = set(need_rerun).union(set(component_name)) - return component_name, force - - @classmethod - def update_job_on_initiator(cls, initiator_job: Job, update_fields: list): - schedule_logger(initiator_job.f_job_id).info(f"try to update job {update_fields} on initiator") - jobs = JobSaver.query_job(job_id=initiator_job.f_job_id) - if not jobs: - raise Exception("Failed to update job status on initiator") - job_info = initiator_job.to_human_model_dict(only_primary_with=update_fields) - for field in update_fields: - job_info[field] = getattr(initiator_job, "f_%s" % field) - for job in jobs: - job_info["role"] = job.f_role - job_info["party_id"] = job.f_party_id - JobSaver.update_job_status(job_info=job_info) - JobSaver.update_job(job_info=job_info) - schedule_logger(initiator_job.f_job_id).info(f"update job {update_fields} on initiator finished") - - @classmethod - def calculate_job_status(cls, task_scheduling_status_code, tasks_status): - # 1. all waiting - # 2. have running - # 3. waiting + end status - # 4. all end status and difference - # 5. all the same end status - tmp_status_set = set(tasks_status) - if TaskStatus.PASS in tmp_status_set: - tmp_status_set.remove(TaskStatus.PASS) - tmp_status_set.add(TaskStatus.SUCCESS) - if len(tmp_status_set) == 1: - # 1 and 5 - return tmp_status_set.pop() - else: - if TaskStatus.RUNNING in tmp_status_set: - # 2 - return JobStatus.RUNNING - if TaskStatus.WAITING in tmp_status_set: - # 3 - if task_scheduling_status_code == SchedulingStatusCode.HAVE_NEXT: - return JobStatus.RUNNING - else: - # have waiting with no next - pass - # have waiting with no next or 4 - for status in sorted(InterruptStatus.status_list(), key=lambda s: StatusSet.get_level(status=s), reverse=True): - if status in tmp_status_set: - return status - if tmp_status_set == {TaskStatus.WAITING, TaskStatus.SUCCESS} and task_scheduling_status_code == SchedulingStatusCode.NO_NEXT: - return JobStatus.CANCELED - - raise Exception("calculate job status failed, all task status: {}".format(tasks_status)) - - @classmethod - def calculate_job_progress(cls, tasks_status): - total = 0 - finished_count = 0 - for task_status in tasks_status.values(): - total += 1 - if EndStatus.contains(task_status): - finished_count += 1 - return total, finished_count - - @classmethod - def stop_job(cls, job_id, role, party_id, stop_status): - schedule_logger(job_id).info(f"request stop job with {stop_status}") - jobs = JobSaver.query_job(job_id=job_id, role=role, party_id=party_id, is_initiator=True) - if len(jobs) > 0: - if stop_status == JobStatus.CANCELED: - schedule_logger(job_id).info("cancel job") - set_cancel_status = schedule_utils.cancel_signal(job_id=job_id, set_or_reset=True) - schedule_logger(job_id).info(f"set job cancel signal {set_cancel_status}") - job = jobs[0] - job.f_status = stop_status - schedule_logger(job_id).info(f"request stop job with {stop_status} to all party") - status_code, response = FederatedScheduler.stop_job(job=jobs[0], stop_status=stop_status) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job_id).info(f"stop job with {stop_status} successfully") - return RetCode.SUCCESS, "success" - else: - initiator_tasks_group = JobSaver.get_tasks_asc(job_id=job.f_job_id, role=job.f_role, party_id=job.f_party_id) - for initiator_task in initiator_tasks_group.values(): - TaskScheduler.collect_task_of_all_party(job, initiator_task=initiator_task, set_status=stop_status) - schedule_logger(job_id).info(f"stop job with {stop_status} failed, {response}") - return RetCode.FEDERATED_ERROR, json_dumps(response) - else: - return RetCode.SUCCESS, "can not found job" - - - @classmethod - @DB.connection_context() - def end_scheduling_updates(cls, job_id): - operate = Job.update({Job.f_end_scheduling_updates: Job.f_end_scheduling_updates + 1}).where(Job.f_job_id == job_id, - Job.f_end_scheduling_updates < JobDefaultConfig.end_status_job_scheduling_updates) - update_status = operate.execute() > 0 - return update_status - - @classmethod - def finish(cls, job, end_status): - schedule_logger(job.f_job_id).info(f"job finished with {end_status}, do something...") - cls.stop_job(job_id=job.f_job_id, role=job.f_initiator_role, party_id=job.f_initiator_party_id, stop_status=end_status) - FederatedScheduler.clean_job(job=job) - schedule_logger(job.f_job_id).info(f"job finished with {end_status}, done") diff --git a/python/fate_flow/detection/detector.py b/python/fate_flow/scheduler/detector.py similarity index 58% rename from python/fate_flow/detection/detector.py rename to python/fate_flow/scheduler/detector.py index a83e3953f..3149f7acd 100644 --- a/python/fate_flow/detection/detector.py +++ b/python/fate_flow/scheduler/detector.py @@ -14,30 +14,21 @@ # limitations under the License. # import time -from typing import List -from fate_arch.common.base_utils import current_timestamp -from fate_arch.session import Session - -from fate_flow.controller.engine_adapt import build_engine -from fate_flow.controller.job_controller import JobController -from fate_flow.controller.task_controller import TaskController -from fate_flow.db.db_models import DB, DependenciesStorageMeta, Job -from fate_flow.db.job_default_config import JobDefaultConfig -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.entity.run_status import ( - EndStatus, FederatedSchedulingStatusCode, - JobStatus, TaskStatus, -) -from fate_flow.manager.dependence_manager import DependenceManager -from fate_flow.manager.resource_manager import ResourceManager -from fate_flow.operation.job_saver import JobSaver -from fate_flow.scheduler.federated_scheduler import FederatedScheduler -from fate_flow.settings import SESSION_VALID_PERIOD -from fate_flow.utils.api_utils import is_localhost +from fate_flow.controller.task import TaskController +from fate_flow.db import Task, Job +from fate_flow.db.base_models import DB +from fate_flow.engine.devices import build_engine +from fate_flow.engine.storage import Session +from fate_flow.entity.types import TaskStatus, JobStatus, EndStatus, LauncherType +from fate_flow.manager.operation.job_saver import JobSaver +from fate_flow.manager.service.resource_manager import ResourceManager +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.controller.federated import FederatedScheduler +from fate_flow.runtime.system_settings import SESSION_VALID_PERIOD +from fate_flow.utils.base_utils import current_timestamp from fate_flow.utils.cron import Cron -from fate_flow.utils.job_utils import check_job_is_timeout, generate_retry_interval -from fate_flow.utils.process_utils import check_process +from fate_flow.utils.job_utils import check_task_is_timeout from fate_flow.utils.log_utils import detect_logger @@ -45,39 +36,48 @@ class Detector(Cron): def run_do(self): self.detect_running_task() self.detect_end_task() - self.detect_running_job() self.detect_resource_record() self.detect_expired_session() - self.detect_dependence_upload_record() - self.detect_deepspeed_task() @classmethod def detect_running_task(cls): detect_logger().info('start to detect running task..') count = 0 try: - running_tasks = JobSaver.query_task(party_status=TaskStatus.RUNNING, run_on_this_party=True) + running_tasks = JobSaver.query_task(party_status=TaskStatus.RUNNING) + detect_logger().info(f'running task: {running_tasks}') stop_job_ids = set() for task in running_tasks: + # check timeout + if check_task_is_timeout(task): + stop_job_ids.add(task.f_job_id) + continue if task.f_run_ip != RuntimeConfig.JOB_SERVER_HOST: cls.detect_cluster_instance_status(task, stop_job_ids) continue - if not task.f_engine_conf or task.f_run_ip != RuntimeConfig.JOB_SERVER_HOST: - continue count += 1 try: - process_exist = build_engine(task.f_engine_conf.get("computing_engine")).is_alive(task) + process_exist = build_engine(task.f_provider_name).is_alive(task) if not process_exist: + # ds task + if task.f_launcher_name == LauncherType.DEEPSPEED: + deepspeed_engine = build_engine(task.f_provider_name, task.f_launcher_name) + if deepspeed_engine.is_alive(task): + continue msg = f"task {task.f_task_id} {task.f_task_version} on {task.f_role} {task.f_party_id}" - detect_logger(job_id=task.f_job_id).error(f"{msg} with {task.f_party_status} process {task.f_run_pid} does not exist") + detect_logger(job_id=task.f_job_id).info( + f"{msg} with {task.f_party_status} process {task.f_run_pid} does not exist") time.sleep(3) - _tasks = JobSaver.query_task(task_id=task.f_task_id, task_version=task.f_task_version, role=task.f_role, party_id=task.f_party_id) + _tasks = JobSaver.query_task(task_id=task.f_task_id, task_version=task.f_task_version, + role=task.f_role, party_id=task.f_party_id) if _tasks: if _tasks[0].f_party_status == TaskStatus.RUNNING: stop_job_ids.add(task.f_job_id) - detect_logger(job_id=task.f_job_id).info(f"{msg} party status has been checked twice, try to stop job") + detect_logger(job_id=task.f_job_id).info( + f"{msg} party status has been checked twice, try to stop job") else: - detect_logger(job_id=task.f_job_id).info(f"{msg} party status has changed to {_tasks[0].f_party_status}, may be stopped by task_controller.stop_task, pass stop job again") + detect_logger(job_id=task.f_job_id).info( + f"{msg} party status has changed to {_tasks[0].f_party_status}, may be stopped by task_controller.stop_task, pass stop job again") else: detect_logger(job_id=task.f_job_id).warning(f"{msg} can not found on db") except Exception as e: @@ -95,36 +95,6 @@ def detect_running_task(cls): finally: detect_logger().info(f"finish detect {count} running task") - @classmethod - def detect_deepspeed_task(cls): - detect_logger().info('start to detect deepspeed running task..') - running_tasks = JobSaver.query_task(party_status=TaskStatus.RUNNING, is_deepspeed=True) - for task in running_tasks: - cls.detect_deepspeed_task_status(task) - detect_logger().info(f'finish detect deepspeed running task {running_tasks}') - - @staticmethod - def detect_deepspeed_task_status(task): - try: - deepspeed_engine = build_engine(task.f_engine_conf.get("computing_engine"), task.f_is_deepspeed) - # query or update - if not deepspeed_engine.is_alive(task): - # update task status to end status - status = deepspeed_engine.query_task_status(task) - detect_logger(task.f_job_id).info(f"task status status: {status}") - task_info = { - "job_id": task.f_job_id, - "task_id": task.f_task_id, - "task_version": task.f_task_version, - "role": task.f_role, - "party_id": task.f_party_id, - "party_status": status - } - TaskController.update_task_status(task_info) - deepspeed_engine.download_log(task) - except Exception as e: - detect_logger(task.f_job_id).exception(e) - @classmethod def detect_end_task(cls): detect_logger().info('start to detect end status task..') @@ -138,12 +108,12 @@ def detect_end_task(cls): ) for task in tasks: try: - if task.f_end_time and task.f_end_time - current_timestamp() < 5 * 60 * 1000: + if task.f_end_time and task.f_end_time - current_timestamp() < 5 * 60 * 1000: continue detect_logger().info(f'start to stop task {task.f_role} {task.f_party_id} {task.f_task_id}' f' {task.f_task_version}') kill_task_status = TaskController.stop_task(task=task, stop_status=TaskStatus.FAILED) - detect_logger().info( f'kill task status: {kill_task_status}') + detect_logger().info(f'kill task status: {kill_task_status}') count += 1 except Exception as e: detect_logger().exception(e) @@ -152,73 +122,6 @@ def detect_end_task(cls): finally: detect_logger().info(f"finish detect {count} end task") - @classmethod - def detect_running_job(cls): - detect_logger().info('start detect running job') - try: - running_jobs = JobSaver.query_job(status=JobStatus.RUNNING, is_initiator=True) - stop_jobs = set() - for job in running_jobs: - try: - if check_job_is_timeout(job): - stop_jobs.add(job) - except Exception as e: - detect_logger(job_id=job.f_job_id).exception(e) - cls.request_stop_jobs(jobs=stop_jobs, stop_msg="running timeout", stop_status=JobStatus.TIMEOUT) - except Exception as e: - detect_logger().exception(e) - finally: - detect_logger().info('finish detect running job') - - @classmethod - @DB.connection_context() - def detect_resource_record(cls): - detect_logger().info('start detect resource recycle') - try: - filter_status = EndStatus.status_list() - filter_status.append(JobStatus.WAITING) - jobs = Job.select().where(Job.f_resource_in_use == True, current_timestamp() - Job.f_apply_resource_time > 10 * 60 * 1000, Job.f_status << filter_status) - stop_jobs = set() - for job in jobs: - if job.f_status == JobStatus.WAITING: - stop_jobs.add(job) - else: - try: - detect_logger(job_id=job.f_job_id).info(f"start to return job {job.f_job_id} on {job.f_role} {job.f_party_id} resource") - flag = ResourceManager.return_job_resource(job_id=job.f_job_id, role=job.f_role, party_id=job.f_party_id) - if flag: - detect_logger(job_id=job.f_job_id).info(f"return job {job.f_job_id} on {job.f_role} {job.f_party_id} resource successfully") - else: - detect_logger(job_id=job.f_job_id).info(f"return job {job.f_job_id} on {job.f_role} {job.f_party_id} resource failed") - except Exception as e: - detect_logger(job_id=job.f_job_id).exception(e) - cls.request_stop_jobs(jobs=stop_jobs, stop_msg="start timeout", stop_status=JobStatus.TIMEOUT) - except Exception as e: - detect_logger().exception(e) - finally: - detect_logger().info('finish detect resource recycle') - - @classmethod - @DB.connection_context() - def detect_dependence_upload_record(cls): - detect_logger().info('start detect dependence upload process') - try: - upload_process_list = DependenciesStorageMeta.select().where(DependenciesStorageMeta.f_upload_status==True) - for dependence in upload_process_list: - if int(dependence.f_pid): - is_alive = check_process(pid=int(dependence.f_pid)) - if not is_alive: - try: - DependenceManager.kill_upload_process(version=dependence.f_version, - storage_engine=dependence.f_storage_engine, - dependence_type=dependence.f_type) - except Exception as e: - detect_logger().exception(e) - except Exception as e: - detect_logger().exception(e) - finally: - detect_logger().info('finish detect dependence upload process') - @classmethod def detect_expired_session(cls): ttl = SESSION_VALID_PERIOD @@ -249,14 +152,16 @@ def detect_expired_session(cls): detect_logger().info('finish detect expired session') @classmethod - def request_stop_jobs(cls, jobs: List[Job], stop_msg, stop_status): + def request_stop_jobs(cls, jobs, stop_msg, stop_status): if not len(jobs): return detect_logger().info(f"have {len(jobs)} should be stopped, because of {stop_msg}") for job in jobs: try: - detect_logger(job_id=job.f_job_id).info(f"detector request start to stop job {job.f_job_id}, because of {stop_msg}") - status = FederatedScheduler.request_stop_job(job=job, stop_status=stop_status) + detect_logger(job_id=job.f_job_id).info( + f"detector request start to stop job {job.f_job_id}, because of {stop_msg}") + status = FederatedScheduler.request_stop_job(job_id=job.f_job_id, party_id=job.f_scheduler_party_id, + stop_status=stop_status) detect_logger(job_id=job.f_job_id).info(f"detector request stop job {job.f_job_id} {status}") except Exception as e: detect_logger(job_id=job.f_job_id).exception(e) @@ -264,7 +169,6 @@ def request_stop_jobs(cls, jobs: List[Job], stop_msg, stop_status): @classmethod def detect_cluster_instance_status(cls, task, stop_job_ids): detect_logger(job_id=task.f_job_id).info('start detect running task instance status') - try: latest_tasks = JobSaver.query_task(task_id=task.f_task_id, role=task.f_role, party_id=task.f_party_id) @@ -290,63 +194,73 @@ def detect_cluster_instance_status(cls, task, stop_job_ids): }) return - if not task.f_run_ip or not task.f_run_port or is_localhost(task.f_run_ip): - return - instance_list = RuntimeConfig.SERVICE_DB.get_servers() instance_list = {instance.http_address for instance_id, instance in instance_list.items()} if f'{task.f_run_ip}:{task.f_run_port}' not in instance_list: - detect_logger(job_id=task.f_job_id).warning( + detect_logger(job_id=task.f_job_id).error( 'detect cluster instance status failed, ' - f'add task {task.f_task_id} {task.f_task_version} to stop list' + 'add task {task.f_task_id} {task.f_task_version} to stop list' ) stop_job_ids.add(task.f_job_id) except Exception as e: detect_logger(job_id=task.f_job_id).exception(e) - -class FederatedDetector(Detector): - def run_do(self): - self.detect_running_job_federated() + @staticmethod + def detect_deepspeed_task_status(task: Task): + try: + deepspeed_engine = build_engine(task.f_provider_name, task.f_launcher_name) + # query or update + if not deepspeed_engine.is_alive(task): + # update task status to end status + status = deepspeed_engine.query_task_status(task) + detect_logger(task.f_job_id).info(f"task status: {status}") + task_info = { + "job_id": task.f_job_id, + "task_id": task.f_task_id, + "task_version": task.f_task_version, + "role": task.f_role, + "party_id": task.f_party_id, + "party_status": status + } + TaskController.update_task_status(task_info) + deepspeed_engine.download_log(task) + except Exception as e: + detect_logger(task.f_job_id).exception(e) @classmethod - def detect_running_job_federated(cls): - detect_logger().info('start federated detect running job') + @DB.connection_context() + def detect_resource_record(cls): + detect_logger().info('start detect resource recycle') try: - running_jobs = JobSaver.query_job(status=JobStatus.RUNNING) + filter_status = EndStatus.status_list() + filter_status.append(JobStatus.WAITING) + jobs = Job.select().where(Job.f_resource_in_use == True, current_timestamp() - Job.f_apply_resource_time > 10 * 60 * 1000, Job.f_status << filter_status) stop_jobs = set() - for job in running_jobs: - cur_retry = 0 - max_retry_cnt = JobDefaultConfig.detect_connect_max_retry_count - long_retry_cnt = JobDefaultConfig.detect_connect_long_retry_count - exception = None - while cur_retry < max_retry_cnt: - detect_logger().info(f"start federated detect running job {job.f_job_id} cur_retry={cur_retry}") + for job in jobs: + if job.f_status == JobStatus.WAITING: + stop_jobs.add(job) + else: try: - status_code, response = FederatedScheduler.connect(job) - if status_code != FederatedSchedulingStatusCode.SUCCESS: - exception = f"connect code: {status_code}" + detect_logger(job_id=job.f_job_id).info(f"start to return job {job.f_job_id} on {job.f_role} {job.f_party_id} resource") + flag = ResourceManager.return_job_resource(job_id=job.f_job_id, role=job.f_role, party_id=job.f_party_id) + if flag: + detect_logger(job_id=job.f_job_id).info(f"return job {job.f_job_id} on {job.f_role} {job.f_party_id} resource successfully") else: - exception = None - detect_logger().info(f"federated detect running job {job.f_job_id} success") - break + detect_logger(job_id=job.f_job_id).info(f"return job {job.f_job_id} on {job.f_role} {job.f_party_id} resource failed") except Exception as e: - exception = e - detect_logger(job_id=job.f_job_id).debug(e) - finally: - retry_interval = generate_retry_interval(cur_retry, max_retry_cnt, long_retry_cnt) - time.sleep(retry_interval) - cur_retry += 1 - if exception is not None: - try: - JobController.stop_jobs(job_id=job.f_job_id, stop_status=JobStatus.FAILED) - except exception as e: - detect_logger().exception(f"stop job failed: {e}") - detect_logger(job.f_job_id).info(f"job {job.f_job_id} connect failed: {exception}") - stop_jobs.add(job) - cls.request_stop_jobs(jobs=stop_jobs, stop_msg="federated error", stop_status=JobStatus.FAILED) + detect_logger(job_id=job.f_job_id).exception(e) + cls.request_stop_jobs(jobs=stop_jobs, stop_msg="start timeout", stop_status=JobStatus.TIMEOUT) except Exception as e: detect_logger().exception(e) finally: - detect_logger().info('finish federated detect running job') + detect_logger().info('finish detect resource recycle') + + +class FederatedDetector(Detector): + def run_do(self): + self.detect_running_job_federated() + + @classmethod + def detect_running_job_federated(cls): + pass diff --git a/python/fate_flow/scheduler/dsl_parser.py b/python/fate_flow/scheduler/dsl_parser.py deleted file mode 100644 index a4d404a5c..000000000 --- a/python/fate_flow/scheduler/dsl_parser.py +++ /dev/null @@ -1,1240 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -# -# -################################################################################ - -# ============================================================================= -# DSL PARSER -# ============================================================================= - -import copy -import json - -from fate_flow.settings import stat_logger -from fate_flow.utils.dsl_exception import DSLNotExistError, ComponentFieldNotExistError, \ - ModuleFieldNotExistError, ComponentInputTypeError, \ - InputComponentNotExistError, InputNameNotExistError, ComponentInputDataTypeError, \ - ComponentInputValueTypeError, \ - ComponentNotExistError, ModeError, DataNotExistInSubmitConfError, ComponentOutputTypeError, \ - ComponentOutputKeyTypeError, LoopError, ComponentMultiMappingError, NamingIndexError, \ - NamingError, NamingFormatError, DeployComponentNotExistError, ModuleNotExistError -from fate_flow.utils.runtime_conf_parse_util import RuntimeConfParserUtil - - -ComponentParameterSource = "ComponentParameterSource" - -class Component(object): - def __init__(self): - self.module = None - self.name = None - self.upstream = [] - self.downstream = [] - self.role_parameters = {} - self.input = {} - self.output = {} - self.component_provider = None - - def copy(self): - copy_obj = Component() - copy_obj.set_module(self.module) - copy_obj.set_name(self.name) - copy_obj.set_input(self.input) - copy_obj.set_downstream(self.downstream) - copy_obj.set_upstream(self.upstream) - copy_obj.set_role_parameters(self.role_parameters) - copy_obj.set_output(self.output) - - return copy_obj - - def set_input(self, inp): - self.input = inp - - def get_input(self): - return self.input - - def set_output(self, output): - self.output = output - - def get_output(self): - return self.output - - def get_module(self): - return self.module - - def set_component_provider(self, interface): - self.component_provider = interface - - def get_component_provider(self): - return self.component_provider - - def get_name(self): - return self.name - - def get_upstream(self): - return self.upstream - - def get_downstream(self): - return self.downstream - - def set_name(self, name): - self.name = name - - def set_module(self, module): - self.module = module - - def set_upstream(self, upstream): - self.upstream = upstream - - def set_downstream(self, downstream): - self.downstream = downstream - - def set_role_parameters(self, role_parameters): - self.role_parameters = role_parameters - - def get_role_parameters(self): - return self.role_parameters - - -class BaseDSLParser(object): - def __init__(self): - self.dsl = None - self.mode = "train" - self.components = [] - self.component_name_index = {} - self.component_upstream = [] - self.component_downstream = [] - self.train_input_model = {} - self.in_degree = [] - self.topo_rank = [] - self.predict_dsl = {} - self.runtime_conf = {} - self.pipeline_runtime_conf = {} - self.graph_dependency = None - self.args_input = None - self.args_data_key = None - self.next_component_to_topo = set() - self.job_parameters = {} - self.provider_cache = {} - self.job_providers = {} - self.version = 2 - self.local_role = None - self.local_party_id = None - self.predict_runtime_conf = {} - - def _init_components(self, mode="train", version=1, **kwargs): - if not self.dsl: - raise DSLNotExistError("") - - components = self.dsl.get("components") - - if components is None: - raise ComponentFieldNotExistError() - - for name in components: - if "module" not in components[name]: - raise ModuleFieldNotExistError(component=name) - - module = components[name]["module"] - - new_component = Component() - new_component.set_name(name) - new_component.set_module(module) - self.component_name_index[name] = len(self.component_name_index) - self.components.append(new_component) - - if version == 2 or mode == "train": - self._check_component_valid_names() - - def _check_component_valid_names(self): - for component in self.components: - name = component.get_name() - for chk in name: - if chk.isalpha() or chk in ["_", "-"] or chk.isdigit(): - continue - else: - raise NamingFormatError(component=name) - - def _find_dependencies(self, mode="train", version=1): - self.component_downstream = [[] for _ in range(len(self.components))] - self.component_upstream = [[] for _ in range(len(self.components))] - - components_details = self.dsl.get("components") - components_output = self._find_outputs(self.dsl) - - for name in self.component_name_index.keys(): - idx = self.component_name_index.get(name) - upstream_input = components_details.get(name).get("input") - downstream_output = components_details.get(name).get("output", {}) - - self.components[idx].set_output(downstream_output) - if upstream_input is None: - continue - elif not isinstance(upstream_input, dict): - raise ComponentInputTypeError(component=name) - else: - self.components[idx].set_input(upstream_input) - - if mode == "train": - input_keywords = {"model": "model", "isometric_model": "model", "cache": "cache"} - else: - input_keywords = {"cache": "cache"} - - for keyword, out_type in input_keywords.items(): - if keyword in upstream_input: - input_list = upstream_input.get(keyword) - - if not isinstance(input_list, list): - raise ComponentInputValueTypeError(component=name, value_type="model", - other_info=input_list) - - for _input in input_list: - input_component = _input.split(".", -1)[0] - input_model_name = _input.split(".")[-1] - if input_component not in self.component_name_index: - raise InputComponentNotExistError(component=name, value_type=keyword, input=input_component) - else: - if input_component not in components_output or out_type not in components_output[input_component]: - raise InputNameNotExistError(component=name, input=input_component, - value_type=keyword, other_info=input_model_name) - - idx_dependency = self.component_name_index.get(input_component) - self.component_downstream[idx_dependency].append(name) - self.component_upstream[idx].append(input_component) - - if keyword == "model" or keyword == "cache": - self.train_input_model[name] = input_component - - if "data" in upstream_input: - data_dict = upstream_input.get("data") - if not isinstance(data_dict, dict): - raise ComponentInputDataTypeError(component=name) - - for data_set in data_dict: - if not isinstance(data_dict.get(data_set), list): - raise ComponentInputValueTypeError(component=name, value_type="data", - other_info=data_dict.get(data_set)) - - if version == 2 and data_set not in ["data", "train_data", "validate_data", "test_data", - "eval_data"]: - stat_logger.warning( - "DSLParser Warning: make sure that input data's data key should be in {}, but {} found".format( - ["data", "train_data", "validate_data", "test_data", "eval_data"], data_set)) - for data_key in data_dict.get(data_set): - input_component = data_key.split(".", -1)[0] - input_data_name = data_key.split(".", -1)[-1] - - if input_component not in self.component_name_index: - raise InputComponentNotExistError(component=name, value_type="data", - input=input_component) - else: - if input_component not in components_output \ - or "data" not in components_output[input_component] \ - or input_data_name not in components_output[input_component]["data"]: - raise InputNameNotExistError(component=name, input=input_component, - value_type="data", other_info=input_data_name) - - idx_dependency = self.component_name_index.get(input_component) - self.component_downstream[idx_dependency].append(name) - self.component_upstream[idx].append(input_component) - - self.in_degree = [0 for _ in range(len(self.components))] - for i in range(len(self.components)): - if self.component_downstream[i]: - self.component_downstream[i] = list(set(self.component_downstream[i])) - - if self.component_upstream[i]: - self.component_upstream[i] = list(set(self.component_upstream[i])) - self.in_degree[self.component_name_index.get(self.components[i].get_name())] = len( - self.component_upstream[i]) - - self._check_dag_dependencies() - - for i in range(len(self.components)): - self.components[i].set_upstream(self.component_upstream[i]) - self.components[i].set_downstream(self.component_downstream[i]) - - def _init_component_setting(self, - component, - provider_detail, - provider_name, - provider_version, - local_role, - local_party_id, - runtime_conf, - redundant_param_check=True, - parse_user_specified_only=False, - previous_parameters=None - ): - """ - init top input - """ - provider = RuntimeConfParserUtil.instantiate_component_provider(provider_detail, - provider_name=provider_name, - provider_version=provider_version) - - pos = self.component_name_index[component] - module = self.components[pos].get_module() - - parent_path = [component] - cur_component = component - isometric_component = None - - while True: - if self.train_input_model.get(cur_component, None) is None: - break - else: - is_warm_start = self._is_warm_start(cur_component) - is_same_module = True - input_component = self.train_input_model.get(cur_component) - input_pos = self.component_name_index[input_component] - if self.components[input_pos].get_module() != module: - is_same_module = False - - if not is_warm_start and is_same_module: - cur_component = self.train_input_model.get(cur_component) - parent_path.append(cur_component) - else: - if (is_warm_start or not is_same_module) and self.components[input_pos].get_module().lower() == "modelloader": - model_load_alias = RuntimeConfParserUtil.get_model_loader_alias(input_component, runtime_conf, - local_role, local_party_id) - isometric_component = model_load_alias - else: - isometric_component = input_component - break - - pre_parameters = {} - if previous_parameters is not None: - if not isometric_component: - pre_parameters = previous_parameters.get(cur_component, {}) - else: - pre_parameters = previous_parameters.get(isometric_component, {}) - - if self.mode == "predict" and pre_parameters: - source_component = previous_parameters.get(component, {}).get(ComponentParameterSource) - if source_component and source_component != cur_component: - runtime_conf = self.runtime_conf - - role_parameters = RuntimeConfParserUtil.get_component_parameters(provider, - runtime_conf, - module, - cur_component, - redundant_param_check=redundant_param_check, - local_role=local_role, - local_party_id=local_party_id, - parse_user_specified_only=parse_user_specified_only, - pre_parameters=pre_parameters) - - if role_parameters: - role_parameters[ComponentParameterSource] = cur_component - for component in parent_path: - idx = self.component_name_index.get(component) - self.components[idx].set_component_provider(provider) - self.components[idx].set_role_parameters(role_parameters) - - return role_parameters - - def _is_warm_start(self, component_name): - component_idx = self.component_name_index.get(component_name) - upstream_inputs = self.components[component_idx].get_input() - if not upstream_inputs: - return False - - return "train_data" in upstream_inputs.get("data", {}) and "model" in upstream_inputs - - def parse_component_parameters(self, component_name, provider_detail, provider_name, provider_version, local_role, - local_party_id, previous_parameters=None): - if self.mode == "predict": - runtime_conf = self.predict_runtime_conf - else: - runtime_conf = self.runtime_conf - - redundant_param_check = True - parameters = self._init_component_setting(component_name, - provider_detail, - provider_name, - provider_version, - local_role, - local_party_id, - runtime_conf, - redundant_param_check, - parse_user_specified_only=False, - previous_parameters=previous_parameters) - - return parameters - - def get_component_info(self, component_name): - if component_name not in self.component_name_index: - raise ComponentNotExistError(component=component_name) - - idx = self.component_name_index.get(component_name) - return self.components[idx] - - def get_upstream_dependent_components(self, component_name): - dependent_component_names = self.get_component_info(component_name).get_upstream() - dependent_components = [] - for up_cpn in dependent_component_names: - up_cpn_idx = self.component_name_index.get(up_cpn) - dependent_components.append(self.components[up_cpn_idx]) - - return dependent_components - - def get_downstream_dependent_components(self, component_name): - component_idx = self.component_name_index.get(component_name) - downstream_components = [] - for cpn in self.component_downstream[component_idx]: - down_cpn_idx = self.component_name_index.get(cpn) - downstream_components.append(self.components[down_cpn_idx]) - - return downstream_components - - def get_topology_components(self): - topo_components = [] - for i in range(len(self.topo_rank)): - topo_components.append(self.components[self.topo_rank[i]]) - - return topo_components - - @staticmethod - def _find_outputs(dsl): - outputs = {} - - components_details = dsl.get("components") - - for name in components_details.keys(): - if "output" not in components_details.get(name): - continue - - component_output = components_details.get(name).get("output") - output_keys = ["data", "model", "cache"] - - if not isinstance(component_output, dict): - raise ComponentOutputTypeError(component=name, other_info=component_output) - - for key in output_keys: - if key not in component_output: - continue - - out_v = component_output.get(key) - if not isinstance(out_v, list): - raise ComponentOutputKeyTypeError(component=name, other_info=key) - - if name not in outputs: - outputs[name] = {} - - outputs[name][key] = out_v - - return outputs - - def _check_dag_dependencies(self): - in_degree = copy.deepcopy(self.in_degree) - stack = [] - - for i in range(len(self.components)): - if in_degree[i] == 0: - stack.append(i) - - tot_nodes = 0 - - while len(stack) > 0: - idx = stack.pop() - tot_nodes += 1 - self.topo_rank.append(idx) - - for down_name in self.component_downstream[idx]: - down_idx = self.component_name_index.get(down_name) - in_degree[down_idx] -= 1 - - if in_degree[down_idx] == 0: - stack.append(down_idx) - - if tot_nodes != len(self.components): - stack = [] - vis = [False for _ in range(len(self.components))] - for i in range(len(self.components)): - if vis[i]: - continue - loops = [] - self._find_loop(i, vis, stack, loops) - raise LoopError(loops) - - def _find_loop(self, u, vis, stack, loops): - vis[u] = True - stack.append(u) - for down_name in self.component_downstream[u]: - if loops: - return - - v = self.component_name_index.get(down_name) - - if v not in stack: - if not vis[v]: - self._find_loop(v, vis, stack, loops) - else: - index = stack.index(v) - for node in stack[index:]: - loops.append(self.components[node].get_name()) - - return - - stack.pop(-1) - - def prepare_graph_dependency_info(self): - dependence_dict = {} - component_module = {} - for component in self.components: - name = component.get_name() - module = component.get_module() - component_module[name] = module - if not component.get_input(): - continue - dependence_dict[name] = [] - inputs = component.get_input() - if "data" in inputs: - data_input = inputs["data"] - for data_key, data_list in data_input.items(): - for dataset in data_list: - up_component_name = dataset.split(".", -1)[0] - up_pos = self.component_name_index.get(up_component_name) - up_component = self.components[up_pos] - data_name = dataset.split(".", -1)[1] - if up_component.get_output().get("data"): - data_pos = up_component.get_output().get("data").index(data_name) - else: - data_pos = 0 - - if data_key == "data" or data_key == "train_data": - data_type = data_key - else: - data_type = "validate_data" - - dependence_dict[name].append({"component_name": up_component_name, - "type": data_type, - "up_output_info": ["data", data_pos]}) - - input_keyword_type_mapping = {"model": "model", - "isometric_model": "model", - "cache": "cache"} - for keyword, v_type in input_keyword_type_mapping.items(): - if keyword in inputs: - input_list = inputs[keyword] - for _input in input_list: - up_component_name = _input.split(".", -1)[0] - if up_component_name == "pipeline": - continue - - link_alias = _input.split(".", -1)[1] - up_pos = self.component_name_index.get(up_component_name) - up_component = self.components[up_pos] - if up_component.get_output().get(v_type): - dep_pos = up_component.get_output().get(v_type).index(link_alias) - else: - dep_pos = 0 - dependence_dict[name].append({"component_name": up_component_name, - "type": v_type, - "up_output_info": [v_type, dep_pos]}) - - if not dependence_dict[name]: - del dependence_dict[name] - - component_list = [None for _ in range(len(self.components))] - topo_rank_reverse_mapping = {} - for i in range(len(self.topo_rank)): - topo_rank_reverse_mapping[self.topo_rank[i]] = i - - for key, value in self.component_name_index.items(): - topo_rank_idx = topo_rank_reverse_mapping[value] - component_list[topo_rank_idx] = key - - base_dependency = {"component_list": component_list, - "dependencies": dependence_dict, - "component_module": component_module, - "component_need_run": {}} - - self.graph_dependency = base_dependency - - def get_dsl_hierarchical_structure(self): - max_depth = [0] * len(self.components) - for idx in range(len(self.topo_rank)): - vertex = self.topo_rank[idx] - for down_name in self.component_downstream[vertex]: - down_vertex = self.component_name_index.get(down_name) - max_depth[down_vertex] = max(max_depth[down_vertex], max_depth[vertex] + 1) - - max_dep = max(max_depth) - hierarchical_structure = [[] for _ in range(max_dep + 1)] - name_component_maps = {} - - for component in self.components: - name = component.get_name() - vertex = self.component_name_index.get(name) - hierarchical_structure[max_depth[vertex]].append(name) - - name_component_maps[name] = component - - return name_component_maps, hierarchical_structure - - def get_dependency(self): - return self.graph_dependency - - def get_dependency_with_parameters(self, component_parameters): - return self.extract_need_run_status(self.graph_dependency, component_parameters) - - def extract_need_run_status(self, graph_dependency, component_parameters): - for rank in range(len(self.topo_rank)): - idx = self.topo_rank[rank] - name = self.components[idx].get_name() - parameters = component_parameters.get(name) - - if not parameters: - graph_dependency["component_need_run"][name] = False - else: - if self.train_input_model.get(name, None) is None: - param_name = "ComponentParam" - if parameters.get(param_name) is None \ - or parameters[param_name].get("need_run") is False: - graph_dependency["component_need_run"][name] = False - else: - graph_dependency["component_need_run"][name] = True - else: - input_model_name = self.train_input_model.get(name) - graph_dependency["component_need_run"][name] = graph_dependency["component_need_run"][ - input_model_name] - - return graph_dependency - - @staticmethod - def verify_dsl(dsl, mode="train"): - dsl_parser = DSLParserV2() - dsl_parser.dsl = dsl - dsl_parser._init_components(mode=mode, version=2) - dsl_parser._find_dependencies(mode=mode, version=2) - - @staticmethod - def verify_dsl_reusability(reused_dsl, new_dsl, reused_components): - # step 1, verify new dsl - dsl_parser = DSLParserV2() - dsl_parser.dsl = new_dsl - dsl_parser._init_components(mode="train", version=2) - dsl_parser._find_dependencies(mode="train", version=2) - - # step 2, verify reused components is a sub-graph - reused_components = set(reused_components) - # reused_components = set(reused_dsl["components"]) & set(new_dsl["components"]) - for cpn in reused_components: - validate_key = ["input", "output", "provider"] - for vk in validate_key: - config_old = reused_dsl["components"][cpn].get(vk, None) - config_new = new_dsl["components"][cpn].get(vk, None) - if config_old != config_new: - raise ValueError(f"Component {cpn}'s {vk} should be same, but old is {config_old}, new is {config_new}") - - inputs = reused_dsl["components"][cpn].get("input", {}) - list_dep_key = ["cache", "model", "isometric_model"] - for dep_key in list_dep_key: - dep_list = inputs.get(dep_key, []) - for dep in dep_list: - input_dep = dep.split(".", -1)[0] - if input_dep not in reused_components: - raise ValueError(f"Component {cpn}'s {dep_key} input {input_dep} should be reused") - - data_dep = inputs.get("data", {}) - for data_key, data_list in data_dep.items(): - for dep in data_list: - input_dep = dep.split(".", -1)[0] - if input_dep not in reused_components: - raise ValueError(f"Component {cpn}'s {data_key} input {input_dep} should be reused") - - @staticmethod - def deploy_component(components, train_dsl, provider_update_dsl=None): - training_cpns = set(train_dsl.get("components").keys()) - deploy_cpns = set(components) - if len(deploy_cpns & training_cpns) != len(deploy_cpns): - raise DeployComponentNotExistError(msg=deploy_cpns - training_cpns) - - dsl_parser = DSLParserV2() - dsl_parser.dsl = train_dsl - dsl_parser._init_components() - dsl_parser._find_dependencies(version=2) - dsl_parser._auto_deduction(deploy_cpns=deploy_cpns, version=2, erase_top_data_input=True) - - """ - dsl_parser.update_predict_dsl_provider(train_dsl) - if provider_update_dsl: - dsl_parser.update_predict_dsl_provider(provider_update_dsl) - """ - return dsl_parser.predict_dsl - - """ - def update_predict_dsl_provider(self, dsl): - for component in dsl["components"]: - provider = dsl["components"][component].get("provider") - if provider and component in self.predict_dsl["components"]: - self.predict_dsl["components"][component]["provider"] = provider - - if "provider" in dsl: - self.predict_dsl["provider"] = dsl["provider"] - """ - - def _auto_deduction(self, deploy_cpns=None, version=1, erase_top_data_input=False): - self.predict_dsl = {"components": {}} - self.predict_components = [] - mapping_list = {} - for i in range(len(self.topo_rank)): - self.predict_components.append(self.components[self.topo_rank[i]].copy()) - mapping_list[self.predict_components[-1].get_name()] = i - - output_data_maps = {} - for i in range(len(self.predict_components)): - name = self.predict_components[i].get_name() - module = self.predict_components[i].get_module() - - if module == "Reader": - if version != 2: - raise ValueError("Reader component can only be set in dsl_version 2") - - if self.get_need_deploy_parameter(name=name, deploy_cpns=deploy_cpns): - self.predict_dsl["components"][name] = {"module": self.predict_components[i].get_module()} - """replace output model to pipeline""" - if "output" in self.dsl["components"][name]: - model_list = self.dsl["components"][name]["output"].get("model", None) - if model_list is not None: - if "input" not in self.predict_dsl["components"][name]: - self.predict_dsl["components"][name]["input"] = {} - - replace_model = [".".join(["pipeline", name, model]) for model in model_list] - self.predict_dsl["components"][name]["input"]["model"] = replace_model - - for out_key, out_val in self.dsl["components"][name]["output"].items(): - if out_val is not None and out_key != "model": - if "output" not in self.predict_dsl["components"][name]: - self.predict_dsl["components"][name]["output"] = {} - - self.predict_dsl["components"][name]["output"][out_key] = out_val - - if "input" in self.dsl["components"][name]: - if "input" not in self.predict_dsl["components"][name]: - self.predict_dsl["components"][name]["input"] = {} - - if "data" in self.dsl["components"][name]["input"]: - self.predict_dsl["components"][name]["input"]["data"] = {} - for data_key, data_value in self._gen_predict_data_mapping(): - if data_key not in self.dsl["components"][name]["input"]["data"]: - continue - - data_set = self.dsl["components"][name]["input"]["data"].get(data_key) - self.predict_dsl["components"][name]["input"]["data"][data_value] = [] - for input_data in data_set: - if version == 1 and input_data.split(".")[0] == "args": - new_input_data = "args.eval_data" - self.predict_dsl["components"][name]["input"]["data"][data_value].append(new_input_data) - elif version == 2 and input_data.split(".")[0] == "args": - self.predict_dsl["components"][name]["input"]["data"][data_value].append(input_data) - elif version == 2 and self.dsl["components"][input_data.split(".")[0]].get("module") == "Reader": - self.predict_dsl["components"][name]["input"]["data"][data_value].append(input_data) - else: - pre_name = input_data.split(".")[0] - data_suffix = input_data.split(".")[1] - if self.get_need_deploy_parameter(name=pre_name, deploy_cpns=deploy_cpns): - self.predict_dsl["components"][name]["input"]["data"][data_value].append(input_data) - else: - self.predict_dsl["components"][name]["input"]["data"][data_value].extend( - output_data_maps[pre_name][data_suffix]) - - break - - if "cache" in self.dsl["components"][name]["input"]: - cache_set = self.dsl["components"][name]["input"]["cache"] - self.predict_dsl["components"][name]["input"]["cache"] = [] - for input_cache in cache_set: - pre_name, cache_suffix = input_cache.split(".")[:2] - input_deploy = self.get_need_deploy_parameter(name=pre_name, deploy_cpns=deploy_cpns) - if version == 1 and not input_deploy: - raise ValueError("In dsl v1, if cache is enabled, input component should be deploy") - self.predict_dsl["components"][name]["input"]["cache"].append(input_cache) - - if version == 2 and erase_top_data_input: - input_dep = {} - for data_key, data_set in self.predict_dsl["components"][name]["input"]["data"].items(): - final_data_set = [] - for input_data in data_set: - cpn_alias = input_data.split(".")[0] - if cpn_alias in self.predict_dsl["components"]: - final_data_set.append(input_data) - - if final_data_set: - input_dep[data_key] = final_data_set - - if not input_dep: - del self.predict_dsl["components"][name]["input"]["data"] - else: - self.predict_dsl["components"][name]["input"]["data"] = input_dep - - else: - name = self.predict_components[i].get_name() - input_data, output_data = None, None - - if "input" in self.dsl["components"][name] and "data" in self.dsl["components"][name]["input"]: - input_data = self.dsl["components"][name]["input"].get("data") - - if "output" in self.dsl["components"][name] and "data" in self.dsl["components"][name]["output"]: - output_data = self.dsl["components"][name]["output"].get("data") - - if output_data is None or input_data is None: - continue - - output_data_maps[name] = {} - for output_data_str in output_data: - if "train_data" in input_data or "eval_data" in input_data or "test_data" in input_data: - if "train_data" in input_data: - up_input_data = input_data.get("train_data")[0] - elif "eval_data" in input_data: - up_input_data = input_data.get("eval_data")[0] - else: - up_input_data = input_data.get("test_data")[0] - elif "data" in input_data: - up_input_data = input_data.get("data")[0] - else: - raise ValueError("train data or eval data or validate data or data should be set") - - up_input_data_component_name = up_input_data.split(".", -1)[0] - if up_input_data_component_name == "args" \ - or self.get_need_deploy_parameter(name=up_input_data_component_name, deploy_cpns=deploy_cpns): - output_data_maps[name][output_data_str] = [up_input_data] - elif self.components[self.component_name_index.get(up_input_data_component_name)].get_module() == "Reader": - output_data_maps[name][output_data_str] = [up_input_data] - else: - up_input_data_suf = up_input_data.split(".", -1)[-1] - output_data_maps[name][output_data_str] = output_data_maps[up_input_data_component_name][up_input_data_suf] - - def run(self, *args, **kwargs): - pass - - def get_runtime_conf(self): - return self.runtime_conf - - def get_dsl(self): - return self.dsl - - def get_args_input(self): - return self.args_input - - @staticmethod - def get_need_deploy_parameter(name, deploy_cpns=None): - if deploy_cpns is not None: - return name in deploy_cpns - - return False - - def get_job_parameters(self, *args, **kwargs): - return self.job_parameters - - def get_job_providers(self, provider_detail=None, dsl=None, conf=None, local_role=None, local_party_id=None): - if dsl is None: - self.job_providers = RuntimeConfParserUtil.get_job_providers(self.dsl, provider_detail, conf, - local_role, local_party_id) - else: - self.job_providers = RuntimeConfParserUtil.get_job_providers(dsl, provider_detail, conf, - local_role, local_party_id) - return self.job_providers - - @staticmethod - def _gen_predict_data_mapping(): - data_mapping = [("data", "data"), ("train_data", "test_data"), - ("validate_data", "test_data"), ("test_data", "test_data")] - - for data_key, data_value in data_mapping: - yield data_key, data_value - - @staticmethod - def generate_predict_conf_template(predict_dsl, train_conf, model_id, model_version): - return RuntimeConfParserUtil.generate_predict_conf_template(predict_dsl, - train_conf, - model_id, - model_version) - - @staticmethod - def get_predict_dsl(predict_dsl=None, module_object_dict=None): - if not predict_dsl: - return {} - - role_predict_dsl = copy.deepcopy(predict_dsl) - component_list = list(predict_dsl.get("components").keys()) - - for component in component_list: - module_object = module_object_dict.get(component) - if module_object: - role_predict_dsl["components"][component]["CodePath"] = module_object - - return role_predict_dsl - - @staticmethod - def get_module_object_name(module, local_role, provider_detail, - provider_name, provider_version): - if not provider_detail: - raise ValueError("Component Providers should be provided") - - provider = RuntimeConfParserUtil.instantiate_component_provider(provider_detail, - provider_name=provider_name, - provider_version=provider_version) - module_obj_name = RuntimeConfParserUtil.get_module_name(role=local_role, - module=module, - provider=provider) - - return module_obj_name - - @staticmethod - def validate_component_param(component, module, runtime_conf, - provider_name, provider_version, provider_detail, - local_role, local_party_id): - provider = RuntimeConfParserUtil.instantiate_component_provider(provider_detail, - provider_name=provider_name, - provider_version=provider_version) - - try: - RuntimeConfParserUtil.get_component_parameters(provider, - runtime_conf, - module, - component, - redundant_param_check=True, - local_role=local_role, - local_party_id=local_party_id, - parse_user_specified_only=False) - return 0 - except Exception as e: - raise ValueError(f"{e}") - - @classmethod - def check_input_existence(cls, dsl): - component_details = dsl.get("components", {}) - component_outputs = cls._find_outputs(dsl) - - input_key = ["data", "model", "isometric_model", "cache"] - non_existence = dict() - for cpn, cpn_detail in component_details.items(): - for k in input_key: - input_deps = cpn_detail.get("input", {}).get(k, {}) - if not input_deps: - continue - - input_splits = None - if k == "data": - for data_k, dep_list in input_deps.items(): - for dep in dep_list: - input_splits = dep.split(".", -1) - - else: - for dep in input_deps: - input_splits = dep.split(".", -1) - if input_splits[0] == "pipeline": - input_splits = input_splits[1:] - - up_cpn, up_link = input_splits - if not component_outputs.get(up_cpn, {}).get(up_link, {}): - if k not in non_existence: - non_existence[k] = list() - non_existence[k].append(f"{cpn}'s {up_cpn}.{up_link}") - - if non_existence: - ret_msg = "non exist input:" - for k, v in non_existence.items(): - ret_msg += f"\n {k}: " + ",".join(v) - - return ret_msg - else: - return "" - - -class DSLParserV1(BaseDSLParser): - def __init__(self): - super(DSLParserV1, self).__init__() - self.version = 1 - - @staticmethod - def get_job_parameters(runtime_conf, conf_version=1): - job_parameters = RuntimeConfParserUtil.get_job_parameters(runtime_conf, - conf_version) - return job_parameters - - @staticmethod - def parse_component_role_parameters(component, dsl, runtime_conf, provider_detail, provider_name, - provider_version): - provider = RuntimeConfParserUtil.instantiate_component_provider(provider_detail, - provider_name=provider_name, - provider_version=provider_version) - - role_parameters = RuntimeConfParserUtil.get_v1_role_parameters(provider, - component, - runtime_conf, - dsl) - - return role_parameters - - @staticmethod - def convert_dsl_v1_to_v2(dsl): - dsl_v2 = copy.deepcopy(dsl) - - # change dsl v1 to dsl v2 - readers = {} - ret_msg = [] - for cpn, cpn_detail in dsl["components"].items(): - new_cpn_detail = copy.deepcopy(cpn_detail) - if cpn_detail.get("input", {}).get("data", {}): - for data_key, dataset in cpn_detail["input"]["data"].items(): - new_dataset = [] - for data in dataset: - up_cpn, up_out_alias = data.split(".", -1) - if up_cpn == "args": - if up_out_alias not in readers: - readers[up_out_alias] = "_".join(["reader", str(len(readers))]) - ret_msg.append(f"{data} is changed to {readers[up_out_alias]}.{up_out_alias}, please " - f"set input data of {readers[up_out_alias]}") - up_link = ".".join([readers[up_out_alias], up_out_alias]) - new_dataset.append(up_link) - else: - new_dataset.append(data) - - new_cpn_detail["input"]["data"][data_key] = new_dataset - - dsl_v2["components"][cpn] = new_cpn_detail - - for output_alias, cpn in readers.items(): - reader_detail = dict(module="Reader", - output={"data": [output_alias]}, - CodePath="Reader") - dsl_v2["components"].update({cpn: reader_detail}) - - return dsl_v2, ", ".join(ret_msg) - - @staticmethod - def convert_conf_v1_to_v2(conf_v1, role_parameters): - conf_v2 = dict() - for attr, conf in conf_v1.items(): - if attr in ["algorithm_parameters", "role_parameters", "job_parameters"]: - continue - - conf_v2[attr] = conf - - job_params = conf_v1.get("job_parameters", {}) - conf_v2["job_parameters"] = dict(common=job_params) - - algorithm_params = conf_v1.get("algorithm_parameters", {}) - if algorithm_params or conf_v1.get("role_parameters"): - conf_v2["component_parameters"] = dict() - if algorithm_params: - conf_v2["component_parameters"]["common"] = algorithm_params - - if conf_v1.get("role_parameters"): - conf_v2["component_parameters"]["role"] = dict() - for cpn, role_params in role_parameters.items(): - conf_v2["component_parameters"]["role"] = RuntimeConfParserUtil.merge_dict(conf_v2["component_parameters"]["role"], - role_params) - - conf_v2["dsl_version"] = 2 - - return conf_v2 - - """ - @staticmethod - def change_conf_v1_to_v2(dsl_v2, conf_v1, provider_detail): - # change conf v1 to conf v2 - readers = dict() - for cpn, cpn_detail in dsl_v2["components"].items(): - if cpn_detail.get("module") != "Reader": - continue - - output_alias = cpn_detail["output"]["data"] - readers[output_alias] = cpn - - conf_v2 = RuntimeConfParserUtil.change_conf_v1_to_v2(dsl_v2, conf_v1, readers, provider_detail) - return conf_v2 - """ - - @staticmethod - def get_components_light_weight(dsl_v2): - components = [] - for cpn, cpn_detail in dsl_v2["components"].items(): - component = Component() - component.set_name(cpn) - component.set_module(cpn_detail["module"]) - components.append(component) - - return components - - -class DSLParserV2(BaseDSLParser): - def __init__(self): - super(DSLParserV2, self).__init__() - self.version = 2 - - def run(self, pipeline_runtime_conf=None, dsl=None, runtime_conf=None, - provider_detail=None, mode="train", - local_role=None, local_party_id=None, *args, **kwargs): - - if mode not in ["train", "predict"]: - raise ModeError("") - - self.dsl = copy.deepcopy(dsl) - self._init_components(mode, version=2) - self._find_dependencies(mode, version=2) - self.runtime_conf = runtime_conf - self.pipeline_runtime_conf = pipeline_runtime_conf - self.mode = mode - self.local_role = local_role - self.local_party_id = local_party_id - - if mode == "train": - self.job_parameters = RuntimeConfParserUtil.get_job_parameters(self.runtime_conf, - conf_version=2) - - else: - """training provider will be delete first""" - pipeline_runtime_conf = copy.deepcopy(pipeline_runtime_conf) - if "provider" in pipeline_runtime_conf: - del pipeline_runtime_conf["provider"] - - predict_runtime_conf = RuntimeConfParserUtil.merge_predict_runtime_conf(pipeline_runtime_conf, - runtime_conf) - self.predict_runtime_conf = predict_runtime_conf - self.job_parameters = RuntimeConfParserUtil.get_job_parameters(predict_runtime_conf, - conf_version=2) - - self.args_input = RuntimeConfParserUtil.get_input_parameters(runtime_conf, - components=self._get_reader_components()) - - self.prepare_graph_dependency_info() - - return self.components - - def parse_user_specified_component_parameters(self, component_name, provider_detail, provider_name, - provider_version, local_role, local_party_id, previous_parameters=None): - if self.mode == "predict": - runtime_conf = self.predict_runtime_conf - else: - runtime_conf = self.runtime_conf - - parameters = self._init_component_setting(component_name, - provider_detail, - provider_name, - provider_version, - local_role, - local_party_id, - runtime_conf, - redundant_param_check=False, - parse_user_specified_only=True, - previous_parameters=previous_parameters) - - return parameters - - def _get_reader_components(self): - reader_components = [] - for cpn, conf in self.dsl.get("components").items(): - if conf.get("module") == "Reader": - reader_components.append(cpn) - - return reader_components - - def get_source_connect_sub_graph(self, valid_nodes): - invalid_nodes = set([self.components[i].get_name() for i in range(len(self.components))]) - set(valid_nodes) - return self._get_source_connect_nodes(invalid_nodes) - - def get_need_revisit_nodes(self, visited_nodes, failed_nodes): - invalid_nodes = set([self.components[i].get_name() for i in range(len(self.components))]) - set(visited_nodes) - invalid_nodes |= set(failed_nodes) - connected_nodes = self._get_source_connect_nodes(invalid_nodes) - connected_nodes_name = [node.get_name() for node in connected_nodes] - revisit_nodes = [] - for node in visited_nodes: - if node not in connected_nodes_name: - idx = self.component_name_index[node] - revisit_nodes.append(self.components[idx]) - - return revisit_nodes - - def _get_source_connect_nodes(self, invalid_nodes): - in_degree = copy.deepcopy(self.in_degree) - stack = [] - for i in range(len(self.components)): - if self.components[i].get_name() in invalid_nodes: - continue - - if in_degree[i] == 0: - stack.append(i) - - connected_nodes = [] - while len(stack) > 0: - idx = stack.pop() - connected_nodes.append(self.components[idx]) - - for down_name in self.component_downstream[idx]: - if down_name in invalid_nodes: - continue - down_idx = self.component_name_index.get(down_name) - in_degree[down_idx] -= 1 - - if in_degree[down_idx] == 0: - stack.append(down_idx) - - return connected_nodes - - @staticmethod - def verify_conf_reusability(reused_conf, new_conf, reused_components): - reused_components = set(reused_components) - - # step1: check role, it should be same - # reused_conf_role = reused_conf.get("role", {}) - # new_conf_role = new_conf.get("role", {}) - # if reused_conf_role != new_conf_role: - # raise ValueError(f"role {reused_conf_role} does not equals to {new_conf_role}") - - # step2: check component common parameters - pre_component_parameters = reused_conf.get("component_parameters", {}) - cur_component_parameters = new_conf.get("component_parameters", {}) - pre_common_params = pre_component_parameters.get("common", {}) - cur_common_params = cur_component_parameters.get("common", {}) - pre_role_params = pre_component_parameters.get("role", {}) - cur_role_params = cur_component_parameters.get("role", {}) - for cpn in reused_components: - cpn_pre_common_params = pre_common_params.get(cpn, {}) - cpn_cur_common_params = cur_common_params.get(cpn, {}) - if cpn_pre_common_params != cpn_cur_common_params: - raise ValueError(f"{cpn}'s common parameters old:{cpn_pre_common_params} != new:{cpn_cur_common_params}") - - # step3: check component role parameters - first_role_params = pre_role_params - second_role_params = cur_role_params - for idx in range(2): - for r, role_params in first_role_params.items(): - for party_idx, params in role_params.items(): - for cpn in reused_components: - cpn_first_role_params = params.get(cpn) - if not cpn_first_role_params: - continue - - cpn_second_role_params = second_role_params.get(r, {}).get(party_idx, {}).get(cpn) - if cpn_first_role_params != cpn_second_role_params: - if idx == 1: - cpn_first_role_params, cpn_second_role_params = cpn_second_role_params, cpn_first_role_params - - raise ValueError(f"{cpn}'s role parameters old:{r}-{party_idx}-{cpn_first_role_params} " - f"!= new: {r}-{party_idx}-{cpn_second_role_params}") - - first_role_params, second_role_params = cur_role_params, pre_role_params - diff --git a/python/fate_flow/scheduler/federated_scheduler.py b/python/fate_flow/scheduler/federated_scheduler.py deleted file mode 100644 index ca183bee7..000000000 --- a/python/fate_flow/scheduler/federated_scheduler.py +++ /dev/null @@ -1,350 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from fate_arch.common import base_utils -from fate_flow.scheduler import SchedulerBase -from fate_flow.utils.api_utils import federated_api -from fate_flow.utils.log_utils import start_log, failed_log, successful_log, warning_log -from fate_flow.utils.log_utils import schedule_logger -from fate_flow.entity import RetCode -from fate_flow.entity.run_status import FederatedSchedulingStatusCode -from fate_flow.entity.types import ResourceOperation -from fate_flow.db.db_models import Job, Task -from fate_flow.operation.job_saver import JobSaver -import threading - -from fate_flow.entity.types import TaskCleanResourceType - - -class FederatedScheduler(SchedulerBase): - """ - Send commands to party, - Report info to initiator - """ - - # Task - REPORT_TO_INITIATOR_FIELDS = ["party_status", "start_time", "update_time", "end_time", "elapsed", "error_report"] - - # Job - @classmethod - def create_job(cls, job: Job): - return cls.job_command(job=job, command="create", command_body=job.to_human_model_dict(), parallel=False) - - @classmethod - def update_parameter(cls, job: Job, updated_parameters): - return cls.job_command(job=job, command="parameter/update", command_body=updated_parameters, parallel=False) - - @classmethod - def resource_for_job(cls, job, operation_type: ResourceOperation, specific_dest=None): - schedule_logger(job.f_job_id).info(f"try to {operation_type} job resource") - status_code, response = cls.job_command(job=job, command=f"resource/{operation_type.value}", specific_dest=specific_dest) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).info(f"{operation_type} job resource successfully") - else: - schedule_logger(job.f_job_id).info(f"{operation_type} job resource failed") - return status_code, response - - @classmethod - def check_component(cls, job, check_type, specific_dest=None): - schedule_logger(job.f_job_id).info(f"try to check component inheritance dependence") - status_code, response = cls.job_command(job=job, command=f"component/{check_type}/check", specific_dest=specific_dest) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).info(f"check job dependence successfully") - else: - schedule_logger(job.f_job_id).info(f"check job dependence failed") - return status_code, response - - @classmethod - def dependence_for_job(cls, job, specific_dest=None): - schedule_logger(job.f_job_id).info(f"try to check job dependence") - status_code, response = cls.job_command(job=job, command=f"dependence/check", specific_dest=specific_dest) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).info(f"check job dependence successfully") - else: - schedule_logger(job.f_job_id).info(f"check job dependence failed") - return status_code, response - - @classmethod - def connect(cls, job): - return cls.job_command(job=job, command="align", command_body={"job_id": job.f_job_id, "role": job.f_role, - "party_id": job.f_party_id}) - - @classmethod - def start_job(cls, job, command_body=None): - return cls.job_command(job=job, command="start", command_body=command_body) - - @classmethod - def align_args(cls, job, command_body): - return cls.job_command(job=job, command="align", command_body=command_body) - - @classmethod - def sync_job(cls, job, update_fields): - sync_info = job.to_human_model_dict(only_primary_with=update_fields) - schedule_logger(job.f_job_id).info("sync job info to all party") - status_code, response = cls.job_command(job=job, command="update", command_body=sync_info) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).info("sync job info to all party successfully") - else: - schedule_logger(job.f_job_id).info(f"sync job info to all party failed: \n{response}") - return status_code, response - - @classmethod - def sync_job_status(cls, job): - schedule_logger(job.f_job_id).info(f"job is {job.f_status}, sync to all party") - status_code, response = cls.job_command(job=job, command=f"status/{job.f_status}", command_body=job.to_human_model_dict()) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).info(f"sync job status {job.f_status} to all party success") - else: - schedule_logger(job.f_job_id).info(f"sync job status {job.f_status} to all party failed: \n{response}") - return status_code, response - - @classmethod - def save_pipelined_model(cls, job): - schedule_logger(job.f_job_id).info("try to save job pipelined model") - status_code, response = cls.job_command(job=job, command="model") - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).info("save job pipelined model success") - else: - schedule_logger(job.f_job_id).info(f"save job pipelined model failed:\n{response}") - return status_code, response - - @classmethod - def stop_job(cls, job, stop_status): - schedule_logger(job.f_job_id).info("try to stop job") - job.f_status = stop_status - status_code, response = cls.job_command(job=job, command="stop/{}".format(stop_status)) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).info("stop job success") - else: - schedule_logger(job.f_job_id).info(f"stop job failed:\n{response}") - return status_code, response - - @classmethod - def request_stop_job(cls, job, stop_status, command_body=None): - return cls.job_command(job=job, command="stop/{}".format(stop_status), dest_only_initiator=True, command_body=command_body) - - @classmethod - def request_rerun_job(cls, job, command_body): - return cls.job_command(job=job, command="rerun", command_body=command_body, dest_only_initiator=True) - - @classmethod - def clean_job(cls, job): - schedule_logger(job.f_job_id).info("try to clean job") - status_code, response = cls.job_command(job=job, command="clean", command_body=job.f_runtime_conf_on_party["role"].copy()) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).info("clean job success") - else: - schedule_logger(job.f_job_id).info(f"clean job failed:\n{response}") - return status_code, response - - @classmethod - def job_command(cls, job, command, command_body=None, dest_only_initiator=False, specific_dest=None, parallel=False): - federated_response = {} - job_parameters = job.f_runtime_conf_on_party["job_parameters"] - if dest_only_initiator: - dest_partis = [(job.f_initiator_role, [job.f_initiator_party_id])] - api_type = "initiator" - elif specific_dest: - dest_partis = specific_dest.items() - api_type = "party" - else: - dest_partis = job.f_roles.items() - api_type = "party" - threads = [] - for dest_role, dest_party_ids in dest_partis: - federated_response[dest_role] = {} - for dest_party_id in dest_party_ids: - endpoint = f"/{api_type}/{job.f_job_id}/{dest_role}/{dest_party_id}/{command}" - args = (job.f_job_id, job.f_role, job.f_party_id, dest_role, dest_party_id, endpoint, command_body, job_parameters["federated_mode"], federated_response) - if parallel: - t = threading.Thread(target=cls.federated_command, args=args) - threads.append(t) - t.start() - else: - cls.federated_command(*args) - for thread in threads: - thread.join() - return cls.return_federated_response(federated_response=federated_response) - - @classmethod - def create_task(cls, job, task): - return cls.task_command(job=job, task=task, command="create", command_body=task.to_human_model_dict()) - - @classmethod - def start_task(cls, job, task): - return cls.task_command(job=job, task=task, command="start", command_body={}, need_user=True) - - @classmethod - def collect_task(cls, job, task): - return cls.task_command(job=job, task=task, command="collect") - - @classmethod - def sync_task(cls, job, task, update_fields): - sync_info = task.to_human_model_dict(only_primary_with=update_fields) - schedule_logger(task.f_job_id).info("sync task {} {} info to all party".format(task.f_task_id, task.f_task_version)) - status_code, response = cls.task_command(job=job, task=task, command="update", command_body=sync_info) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(task.f_job_id).info("sync task {} {} info to all party successfully".format(task.f_task_id, task.f_task_version)) - else: - schedule_logger(task.f_job_id).info("sync task {} {} info to all party failed: \n{}".format(task.f_task_id, task.f_task_version, response)) - return status_code, response - - @classmethod - def sync_task_status(cls, job, task): - schedule_logger(task.f_job_id).info("task {} {} is {}, sync to all party".format(task.f_task_id, task.f_task_version, task.f_status)) - status_code, response = cls.task_command(job=job, task=task, command=f"status/{task.f_status}") - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(task.f_job_id).info("sync task {} {} status {} to all party success".format(task.f_task_id, task.f_task_version, task.f_status)) - else: - schedule_logger(task.f_job_id).info("sync task {} {} status {} to all party failed: \n{}".format(task.f_task_id, task.f_task_version, task.f_status, response)) - return status_code, response - - @classmethod - def stop_task(cls, job, task, stop_status, command_body=None): - schedule_logger(task.f_job_id).info("try to stop task {} {}".format(task.f_task_id, task.f_task_version)) - task.f_status = stop_status - status_code, response = cls.task_command(job=job, task=task, command="stop/{}".format(stop_status), command_body=command_body) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).info("stop task {} {} success".format(task.f_task_id, task.f_task_version)) - else: - schedule_logger(job.f_job_id).info("stop task {} {} failed:\n{}".format(task.f_task_id, task.f_task_version, response)) - return status_code, response - - @classmethod - def clean_task(cls, job, task, content_type: TaskCleanResourceType): - schedule_logger(task.f_job_id).info("try to clean task {} {} {}".format(task.f_task_id, task.f_task_version, content_type)) - status_code, response = cls.task_command(job=job, task=task, command="clean/{}".format(content_type.value)) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).info("clean task {} {} {} successfully".format(task.f_task_id, task.f_task_version, content_type)) - else: - schedule_logger(job.f_job_id).info("clean task {} {} {} failed:\n{}".format(task.f_task_id, task.f_task_version, content_type, response)) - return status_code, response - - @classmethod - def task_command(cls, job: Job, task: Task, command, command_body=None, parallel=False, need_user=False): - msg = f"execute federated task {task.f_component_name} command({command})" - federated_response = {} - job_parameters = job.f_runtime_conf_on_party["job_parameters"] - tasks = JobSaver.query_task(task_id=task.f_task_id, only_latest=True) - threads = [] - for task in tasks: - dest_role, dest_party_id = task.f_role, task.f_party_id - federated_response[dest_role] = federated_response.get(dest_role, {}) - endpoint = f"/party/{task.f_job_id}/{task.f_component_name}/{task.f_task_id}/{task.f_task_version}/{dest_role}/{dest_party_id}/{command}" - if need_user: - command_body["user_id"] = job.f_user.get(dest_role, {}).get(str(dest_party_id), "") - schedule_logger(job.f_job_id).info(f'user:{job.f_user}, dest_role:{dest_role}, dest_party_id:{dest_party_id}') - schedule_logger(job.f_job_id).info(f'command_body: {command_body}') - args = (job.f_job_id, job.f_role, job.f_party_id, dest_role, dest_party_id, endpoint, command_body, job_parameters["federated_mode"], federated_response) - if parallel: - t = threading.Thread(target=cls.federated_command, args=args) - threads.append(t) - t.start() - else: - cls.federated_command(*args) - for thread in threads: - thread.join() - status_code, response = cls.return_federated_response(federated_response=federated_response) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).info(successful_log(msg)) - elif status_code == FederatedSchedulingStatusCode.NOT_EFFECTIVE: - schedule_logger(job.f_job_id).warning(warning_log(msg)) - elif status_code == FederatedSchedulingStatusCode.ERROR: - schedule_logger(job.f_job_id).critical(failed_log(msg, detail=response)) - else: - schedule_logger(job.f_job_id).error(failed_log(msg, detail=response)) - return status_code, response - - @classmethod - def federated_command(cls, job_id, src_role, src_party_id, dest_role, dest_party_id, endpoint, body, federated_mode, federated_response): - st = base_utils.current_timestamp() - log_msg = f"sending {endpoint} federated command" - schedule_logger(job_id).info(start_log(msg=log_msg)) - try: - response = federated_api(job_id=job_id, - method='POST', - endpoint=endpoint, - src_role=src_role, - src_party_id=src_party_id, - dest_party_id=dest_party_id, - json_body=body if body else {}, - federated_mode=federated_mode) - except Exception as e: - schedule_logger(job_id=job_id).exception(e) - response = { - "retcode": RetCode.FEDERATED_ERROR, - "retmsg": "Federated schedule error, {}".format(e) - } - if response["retcode"] != RetCode.SUCCESS: - if response["retcode"] in [RetCode.NOT_EFFECTIVE, RetCode.RUNNING]: - schedule_logger(job_id).warning(warning_log(msg=log_msg, role=dest_role, party_id=dest_party_id)) - else: - schedule_logger(job_id).error(failed_log(msg=log_msg, role=dest_role, party_id=dest_party_id, detail=response["retmsg"])) - federated_response[dest_role][dest_party_id] = response - et = base_utils.current_timestamp() - schedule_logger(job_id).info(f"{log_msg} use {et - st} ms") - - @classmethod - def report_task_to_initiator(cls, task: Task): - """ - :param task: - :return: - """ - if task.f_role != task.f_initiator_role and task.f_party_id != task.f_initiator_party_id: - try: - response = federated_api(job_id=task.f_job_id, - method='POST', - endpoint='/initiator/{}/{}/{}/{}/{}/{}/report'.format( - task.f_job_id, - task.f_component_name, - task.f_task_id, - task.f_task_version, - task.f_role, - task.f_party_id), - src_party_id=task.f_party_id, - dest_party_id=task.f_initiator_party_id, - src_role=task.f_role, - json_body=task.to_human_model_dict(only_primary_with=cls.REPORT_TO_INITIATOR_FIELDS), - federated_mode=task.f_federated_mode) - except Exception as e: - schedule_logger(task.f_job_id).error(f"report task to initiator error: {e}") - return False - if response["retcode"] != RetCode.SUCCESS: - retmsg = response["retmsg"] - schedule_logger(task.f_job_id).error(f"report task to initiator error: {retmsg}") - return False - else: - return True - else: - return False - - @classmethod - def tracker_command(cls, job, request_data, command, json_body=None): - job_parameters = job.f_runtime_conf_on_party["job_parameters"] - response = federated_api(job_id=str(request_data['job_id']), - method='POST', - endpoint='/tracker/{}/{}/{}/{}/{}'.format( - request_data['job_id'], - request_data['component_name'], - request_data['role'], - request_data['party_id'], - command), - src_party_id=job.f_party_id, - dest_party_id=request_data['party_id'], - src_role=job.f_role, - json_body=json_body if json_body else {}, - federated_mode=job_parameters["federated_mode"]) - return response diff --git a/python/fate_flow/scheduler/scheduler.py b/python/fate_flow/scheduler/scheduler.py new file mode 100644 index 000000000..bfc3a44bd --- /dev/null +++ b/python/fate_flow/scheduler/scheduler.py @@ -0,0 +1,591 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import abc +import os + +from pydantic import typing + +from fate_flow.controller.task import TaskController +from fate_flow.entity.code import SchedulingStatusCode, FederatedSchedulingStatusCode +from fate_flow.entity.spec.dag import DAGSchema +from fate_flow.db.schedule_models import ScheduleJob, ScheduleTaskStatus +from fate_flow.entity.types import StatusSet, JobStatus, TaskStatus, EndStatus, InterruptStatus, ResourceOperation, \ + FederatedCommunicationType, AutoRerunStatus +from fate_flow.entity.code import ReturnCode +from fate_flow.errors.server_error import NoFoundJob +from fate_flow.controller.parser import JobParser +from fate_flow.manager.operation.job_saver import ScheduleJobSaver, JobSaver +from fate_flow.runtime.job_default_config import JobDefaultConfig +from fate_flow.controller.federated import FederatedScheduler +from fate_flow.utils import schedule_utils, wraps_utils, job_utils +from fate_flow.utils.base_utils import json_dumps +from fate_flow.utils.cron import Cron +from fate_flow.utils.log_utils import schedule_logger, exception_to_trace_string + + +class SchedulerABC(Cron): + @abc.abstractmethod + def run_do(self): + """ + description： + Scheduling various status job, including: waiting、running、ready、rerun、end、etc. + """ + + @classmethod + def stop_job(cls, job_id: str, stop_status: str): + """ + description： + Stop a job to all parties and set the job status to end status + :param job_id: job id + :param stop_status: In which state to stop the task. + + """ + + @classmethod + def rerun_job(cls, job_id: str, auto: bool, tasks=None): + """ + description： + rerun a job + :param job_id: job id + :param auto: Whether the scheduler automatically rerun + :param tasks: Specified rerun task list. + + """ + + +class DAGScheduler(SchedulerABC): + @classmethod + def dag_parser(cls, dag): + return JobParser(dag) + + def run_do(self): + # waiting + schedule_logger().info("start schedule waiting jobs") + # order by create_time and priority + jobs = ScheduleJobSaver.query_job( + status=JobStatus.WAITING, + order_by=["priority", "create_time"], + reverse=[True, False] + ) + schedule_logger().info(f"have {len(jobs)} waiting jobs") + if len(jobs): + job = jobs[0] + schedule_logger().info(f"schedule waiting job {job.f_job_id}") + try: + self.schedule_waiting_jobs(job=job, lock=True) + except Exception as e: + schedule_logger(job.f_job_id).exception(e) + schedule_logger(job.f_job_id).error("schedule waiting job failed") + schedule_logger().info("schedule waiting jobs finished") + + # running + schedule_logger().info("start schedule running jobs") + jobs = ScheduleJobSaver.query_job(status=JobStatus.RUNNING, order_by="create_time", reverse=False) + schedule_logger().info(f"have {len(jobs)} running jobs") + for job in jobs: + schedule_logger().info(f"schedule running job {job.f_job_id}") + try: + self.schedule_running_job(job=job, lock=True) + except Exception as e: + schedule_logger(job.f_job_id).exception(e) + schedule_logger(job.f_job_id).error("schedule job failed") + schedule_logger().info("schedule running jobs finished") + + # rerun + schedule_logger().info("start schedule rerun jobs") + jobs = ScheduleJobSaver.query_job(rerun_signal=True, order_by="create_time", reverse=False) + schedule_logger().info(f"have {len(jobs)} rerun jobs") + for job in jobs: + schedule_logger(job.f_job_id).info(f"schedule rerun job {job.f_job_id}") + try: + self.schedule_rerun_job(job=job, lock=True) + except Exception as e: + schedule_logger(job.f_job_id).exception(e) + schedule_logger(job.f_job_id).error("schedule job failed") + schedule_logger().info("schedule rerun jobs finished") + + @classmethod + def apply_job_resource(cls, job): + apply_status_code, federated_response = FederatedScheduler.resource_for_job( + job_id=job.f_job_id, + roles=job.f_parties, + operation_type=ResourceOperation.APPLY.value + ) + if apply_status_code == FederatedSchedulingStatusCode.SUCCESS: + return True + else: + cls.rollback_job_resource(job, federated_response) + return False + + @classmethod + def rollback_job_resource(cls, job, federated_response): + rollback_party = [] + failed_party = [] + for dest_role in federated_response.keys(): + for dest_party_id in federated_response[dest_role].keys(): + retcode = federated_response[dest_role][dest_party_id]["code"] + if retcode == ReturnCode.Base.SUCCESS: + rollback_party.append({"role": dest_role, "party_id": [dest_party_id]}) + else: + failed_party.append({"role": dest_role, "party_id": [dest_party_id]}) + schedule_logger(job.f_job_id).info("job apply resource failed on {}, rollback {}".format(failed_party, + rollback_party)) + if rollback_party: + return_status_code, federated_response = FederatedScheduler.resource_for_job( + job_id=job.f_job_id, + roles=rollback_party, + operation_type=ResourceOperation.RETURN.value + ) + if return_status_code != FederatedSchedulingStatusCode.SUCCESS: + schedule_logger(job.f_job_id).info(f"job return resource failed:\n{federated_response}") + else: + schedule_logger(job.f_job_id).info("job no party should be rollback resource") + + @classmethod + @wraps_utils.schedule_lock + def schedule_waiting_jobs(cls, job: ScheduleJob): + if job.f_cancel_signal: + FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_parties, + job_info={"job_id": job.f_job_id, "status": JobStatus.CANCELED}) + ScheduleJobSaver.update_job_status({"job_id": job.f_job_id, "status": JobStatus.CANCELED}) + schedule_logger(job.f_job_id).info("job have cancel signal") + return + status = cls.apply_job_resource(job) + if status: + cls.start_job(job_id=job.f_job_id, roles=job.f_parties) + + @wraps_utils.schedule_lock + def schedule_running_job(self, job: ScheduleJob, force_sync_status=False): + schedule_logger(job.f_job_id).info("scheduling running job") + task_scheduling_status_code, auto_rerun_tasks, tasks = TaskScheduler.schedule(job=job) + tasks_status = dict([(task.f_task_name, task.f_status) for task in tasks]) + schedule_logger(job_id=job.f_job_id).info(f"task_scheduling_status_code: {task_scheduling_status_code}, " + f"tasks_status: {tasks_status.values()}") + new_job_status = self.calculate_job_status(task_scheduling_status_code=task_scheduling_status_code, + tasks_status=tasks_status.values()) + if new_job_status == JobStatus.WAITING and job.f_cancel_signal: + new_job_status = JobStatus.CANCELED + total, finished_count = self.calculate_job_progress(tasks_status=tasks_status) + new_progress = float(finished_count) / total * 100 + schedule_logger(job.f_job_id).info( + f"job status is {new_job_status}, calculate by task status list: {tasks_status}") + if new_job_status != job.f_status or new_progress != job.f_progress: + # Make sure to update separately, because these two fields update with anti-weight logic + if int(new_progress) - job.f_progress > 0: + job.f_progress = new_progress + FederatedScheduler.update_job(job_id=job.f_job_id, + roles=job.f_parties, + command_body={"job_id": job.f_job_id, "progress": job.f_progress}) + self.update_job_on_scheduler(schedule_job=job, update_fields=["progress"]) + if new_job_status != job.f_status: + job.f_status = new_job_status + FederatedScheduler.sync_job_status( + job_id=job.f_job_id, roles=job.f_parties, + job_info={"job_id": job.f_job_id, "status": new_job_status} + ) + self.update_job_on_scheduler(schedule_job=job, update_fields=["status"]) + if EndStatus.contains(job.f_status): + self.finish(job=job, end_status=job.f_status) + if auto_rerun_tasks: + schedule_logger(job.f_job_id).info("job have auto rerun tasks") + self.rerun_job(job_id=job.f_job_id, tasks=auto_rerun_tasks, auto=True) + if force_sync_status: + FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_roles, status=job.f_status, + job_info=job.to_human_model_dict()) + schedule_logger(job.f_job_id).info("finish scheduling running job") + + @wraps_utils.schedule_lock + def schedule_rerun_job(self, job): + if EndStatus.contains(job.f_status): + job.f_status = JobStatus.WAITING + schedule_logger(job.f_job_id).info("job has been finished, set waiting to rerun") + status, response = FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_parties, + job_info={"job_id": job.f_job_id, + "status": job.f_status}) + if status == FederatedSchedulingStatusCode.SUCCESS: + schedule_utils.rerun_signal(job_id=job.f_job_id, set_or_reset=False) + schedule_logger(job.f_job_id).info("job set waiting to rerun successfully") + else: + schedule_logger(job.f_job_id).warning("job set waiting to rerun failed") + ScheduleJobSaver.update_job_status({"job_id": job.f_job_id, "status": job.f_status}) + else: + schedule_utils.rerun_signal(job_id=job.f_job_id, set_or_reset=False) + self.schedule_running_job(job) + + @classmethod + def calculate_job_status(cls, task_scheduling_status_code, tasks_status): + tmp_status_set = set(tasks_status) + if TaskStatus.PASS in tmp_status_set: + tmp_status_set.remove(TaskStatus.PASS) + tmp_status_set.add(TaskStatus.SUCCESS) + if len(tmp_status_set) == 1: + return tmp_status_set.pop() + else: + if TaskStatus.RUNNING in tmp_status_set: + return JobStatus.RUNNING + if TaskStatus.WAITING in tmp_status_set: + if task_scheduling_status_code == SchedulingStatusCode.HAVE_NEXT: + return JobStatus.RUNNING + else: + pass + for status in sorted(InterruptStatus.status_list(), key=lambda s: StatusSet.get_level(status=s), + reverse=True): + if status in tmp_status_set: + return status + if tmp_status_set == {TaskStatus.WAITING, + TaskStatus.SUCCESS} and task_scheduling_status_code == SchedulingStatusCode.NO_NEXT: + return JobStatus.CANCELED + + raise Exception("calculate job status failed, all task status: {}".format(tasks_status)) + + @classmethod + def calculate_job_progress(cls, tasks_status): + total = 0 + finished_count = 0 + for task_status in tasks_status.values(): + total += 1 + if EndStatus.contains(task_status): + finished_count += 1 + return total, finished_count + + @classmethod + def start_job(cls, job_id, roles): + schedule_logger(job_id).info(f"start job {job_id}") + status_code, response = FederatedScheduler.start_job(job_id, roles) + schedule_logger(job_id).info(f"start job {job_id} status code: {status_code}, response: {response}") + ScheduleJobSaver.update_job_status(job_info={"job_id": job_id, "status": StatusSet.RUNNING}) + + @classmethod + def stop_job(cls, job_id, stop_status): + schedule_logger(job_id).info(f"request stop job with {stop_status}") + jobs = ScheduleJobSaver.query_job(job_id=job_id) + if len(jobs) > 0: + if stop_status == JobStatus.CANCELED: + schedule_logger(job_id).info("cancel job") + set_cancel_status = schedule_utils.cancel_signal(job_id=job_id, set_or_reset=True) + schedule_logger(job_id).info(f"set job cancel signal {set_cancel_status}") + job = jobs[0] + job.f_status = stop_status + schedule_logger(job_id).info(f"request stop job with {stop_status} to all party") + status_code, response = FederatedScheduler.stop_job(job_id=job_id, roles=job.f_parties) + if status_code == FederatedSchedulingStatusCode.SUCCESS: + schedule_logger(job_id).info(f"stop job with {stop_status} successfully") + return ReturnCode.Base.SUCCESS, "success" + else: + tasks_group = ScheduleJobSaver.get_status_tasks_asc(job_id=job.f_job_id) + for task in tasks_group.values(): + TaskScheduler.collect_task_of_all_party(job, task=task, set_status=stop_status) + schedule_logger(job_id).info(f"stop job with {stop_status} failed, {response}") + return ReturnCode.Job.KILL_FAILED, json_dumps(response) + else: + raise NoFoundJob(job_id=job_id) + + @classmethod + def update_job_on_scheduler(cls, schedule_job: ScheduleJob, update_fields: list): + schedule_logger(schedule_job.f_job_id).info(f"try to update job {update_fields} on scheduler") + jobs = ScheduleJobSaver.query_job(job_id=schedule_job.f_job_id) + if not jobs: + raise Exception("Failed to update job status on scheduler") + job_info = schedule_job.to_human_model_dict(only_primary_with=update_fields) + for field in update_fields: + job_info[field] = getattr(schedule_job, "f_%s" % field) + if "status" in update_fields: + ScheduleJobSaver.update_job_status(job_info=job_info) + ScheduleJobSaver.update_job(job_info=job_info) + schedule_logger(schedule_job.f_job_id).info(f"update job {update_fields} on scheduler finished") + + @classmethod + def rerun_job(cls, job_id, auto, tasks: typing.List[ScheduleTaskStatus] = None): + schedule_logger(job_id).info(f"try to rerun job {job_id}") + jobs = ScheduleJobSaver.query_job(job_id=job_id) + if not jobs: + raise RuntimeError(f"can not found job {job_id}") + job = jobs[0] + if tasks: + schedule_logger(job_id).info(f"require {[task.f_task_name for task in tasks]} to rerun") + else: + # todo: get_need_revisit_nodes + tasks = ScheduleJobSaver.query_task(job_id=job_id, status=TaskStatus.CANCELED, scheduler_status=True) + job_can_rerun = any([TaskController.prepare_rerun_task( + job=job, task=task, auto=auto, force=False, + ) for task in tasks]) + schedule_logger(job_id).info("job set rerun signal") + status = schedule_utils.rerun_signal(job_id=job_id, set_or_reset=True) + schedule_logger(job_id).info(f"job set rerun signal {'successfully' if status else 'failed'}") + return True + + @classmethod + def finish(cls, job, end_status): + schedule_logger(job.f_job_id).info(f"job finished with {end_status}, do something...") + cls.stop_job(job_id=job.f_job_id, stop_status=end_status) + # todo: clean job + schedule_logger(job.f_job_id).info(f"job finished with {end_status}, done") + + @classmethod + def create_all_job(cls, dag, job_id=None): + dag_schema = DAGSchema(**dag) + if not job_id: + job_id = job_utils.generate_job_id() + schedule_logger(job_id).info( + f"submit job, dag {dag_schema.dag.dict()}, schema version {dag_schema.schema_version}") + submit_result = { + "job_id": job_id, + "data": {} + } + try: + job = ScheduleJob() + job.f_job_id = job_id + job.f_parties = [party.dict() for party in dag_schema.dag.parties] + job.f_initiator_party_id = dag_schema.dag.conf.initiator_party_id + job.f_scheduler_party_id = dag_schema.dag.conf.scheduler_party_id + if dag_schema.dag.conf.priority: + job.f_priority = dag_schema.dag.conf.priority + cls.fill_default_job_parameters(job_id, dag_schema) + job.f_dag = dag_schema.dict() + submit_result["data"].update({ + "model_id": dag_schema.dag.conf.model_id, + "model_version": dag_schema.dag.conf.model_version + }) + job.f_status = StatusSet.READY + ScheduleJobSaver.create_job(job.to_human_model_dict()) + body = dag_schema.dict() + body.update({ + "job_id": job_id + }) + status_code, response = FederatedScheduler.create_job( + job_id, job.f_parties, job.f_initiator_party_id, body + ) + if status_code != FederatedSchedulingStatusCode.SUCCESS: + job.f_status = JobStatus.FAILED + FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_parties, job_info={ + "job_id": job.f_job_id, + "status": job.f_status + }) + raise Exception("create job failed", response) + else: + job.f_status = JobStatus.WAITING + TaskController.create_schedule_tasks(job, dag_schema) + status_code, response = FederatedScheduler.sync_job_status(job_id=job.f_job_id, roles=job.f_parties, + job_info={"job_id": job.f_job_id, + "status": job.f_status}) + if status_code != FederatedSchedulingStatusCode.SUCCESS: + raise Exception(f"set job to waiting status failed: {response}") + ScheduleJobSaver.update_job_status({"job_id": job.f_job_id, "status": job.f_status}) + schedule_logger(job_id).info(f"submit job successfully, job id is {job.f_job_id}") + result = { + "code": ReturnCode.Base.SUCCESS, + "message": "success" + } + submit_result.update(result) + except Exception as e: + schedule_logger(job_id).exception(e) + submit_result["code"] = ReturnCode.Job.CREATE_JOB_FAILED + submit_result["message"] = exception_to_trace_string(e) + return submit_result + + @classmethod + def fill_default_job_parameters(cls, job_id: str, dag_schema: DAGSchema): + if not dag_schema.dag.conf.sync_type: + dag_schema.dag.conf.sync_type = JobDefaultConfig.sync_type + if not dag_schema.dag.conf.model_id or not dag_schema.dag.conf.model_id: + dag_schema.dag.conf.model_id, dag_schema.dag.conf.model_version = job_utils.generate_model_info(job_id) + if not dag_schema.dag.conf.auto_retries: + dag_schema.dag.conf.auto_retries = JobDefaultConfig.auto_retries + + +class TaskScheduler(object): + @classmethod + def schedule(cls, job): + schedule_logger(job.f_job_id).info("scheduling job tasks") + dag_schema = DAGSchema(**job.f_dag) + job_parser = JobParser(DAGSchema(**job.f_dag)) + tasks_group = ScheduleJobSaver.get_status_tasks_asc(job_id=job.f_job_id) + waiting_tasks = {} + auto_rerun_tasks = [] + job_interrupt = False + canceled = job.f_cancel_signal + for task in tasks_group.values(): + if task.f_sync_type == FederatedCommunicationType.POLL: + cls.collect_task_of_all_party(job=job, task=task) + else: + pass + new_task_status = cls.get_federated_task_status(job_id=task.f_job_id, task_id=task.f_task_id, + task_version=task.f_task_version) + task_interrupt = False + task_status_have_update = False + if new_task_status != task.f_status: + task_status_have_update = True + schedule_logger(job.f_job_id).info(f"sync task status {task.f_status} to {new_task_status}") + task.f_status = new_task_status + FederatedScheduler.sync_task_status(task_id=task.f_task_id, command_body={"status": task.f_status}) + ScheduleJobSaver.update_task_status(task.to_human_model_dict(), scheduler_status=True) + if InterruptStatus.contains(new_task_status): + task_interrupt = True + job_interrupt = True + if task.f_status == TaskStatus.WAITING: + waiting_tasks[task.f_task_name] = task + elif task_status_have_update and EndStatus.contains(task.f_status) or task_interrupt: + schedule_logger(task.f_job_id).info(f"stop task with status: {task.f_status}") + FederatedScheduler.stop_task(task_id=task.f_task_id, command_body={"status": task.f_status}) + if not canceled and AutoRerunStatus.contains(task.f_status): + if task.f_auto_retries > 0: + auto_rerun_tasks.append(task) + schedule_logger(job.f_job_id).info(f"task {task.f_task_id} {task.f_status} will be retried") + else: + schedule_logger(job.f_job_id).info(f"task {task.f_task_id} {task.f_status} has no retry count") + + scheduling_status_code = SchedulingStatusCode.NO_NEXT + schedule_logger(job.f_job_id).info(f"canceled status {canceled}, job interrupt status {job_interrupt}") + if not canceled and not job_interrupt: + for task_id, waiting_task in waiting_tasks.items(): + dependent_tasks = job_parser.infer_dependent_tasks( + dag_schema.dag.tasks[waiting_task.f_task_name].inputs + ) + schedule_logger(job.f_job_id).info(f"task {waiting_task.f_task_name} dependent tasks:{dependent_tasks}") + for task_name in dependent_tasks: + dependent_task = tasks_group[task_name] + if dependent_task.f_status != TaskStatus.SUCCESS: + break + else: + scheduling_status_code = SchedulingStatusCode.HAVE_NEXT + status_code = cls.start_task(job=job, task=waiting_task) + if status_code == SchedulingStatusCode.NO_RESOURCE: + schedule_logger(job.f_job_id).info( + f"task {waiting_task.f_task_id} can not apply resource, wait for the next round of scheduling") + break + elif status_code == SchedulingStatusCode.FAILED: + schedule_logger(job.f_job_id).info(f"task status code: {status_code}") + scheduling_status_code = SchedulingStatusCode.FAILED + waiting_task.f_status = StatusSet.FAILED + FederatedScheduler.sync_task_status(task_id=waiting_task.f_task_id, command_body={ + "status": waiting_task.f_status}) + break + else: + schedule_logger(job.f_job_id).info("have cancel signal, pass start job tasks") + schedule_logger(job.f_job_id).info("finish scheduling job tasks") + return scheduling_status_code, auto_rerun_tasks, tasks_group.values() + + @classmethod + def start_task(cls, job, task): + schedule_logger(task.f_job_id).info("try to start task {} {}".format(task.f_task_id, task.f_task_version)) + # apply resource for task + apply_status = cls.apply_task_resource(task, job) + if not apply_status: + return SchedulingStatusCode.NO_RESOURCE + task.f_status = TaskStatus.RUNNING + ScheduleJobSaver.update_task_status( + task_info=task.to_human_model_dict(only_primary_with=["status"]), scheduler_status=True + ) + schedule_logger(task.f_job_id).info("start task {} {}".format(task.f_task_id, task.f_task_version)) + FederatedScheduler.sync_task_status(task_id=task.f_task_id, command_body={"status": task.f_status}) + ScheduleJobSaver.update_task_status(task.to_human_model_dict(), scheduler_status=True) + status_code, response = FederatedScheduler.start_task(task_id=task.f_task_id) + if status_code == FederatedSchedulingStatusCode.SUCCESS: + return SchedulingStatusCode.SUCCESS + else: + return SchedulingStatusCode.FAILED + + @classmethod + def apply_task_resource(cls, task, job): + apply_status_code, federated_response = FederatedScheduler.resource_for_task( + task_id=task.f_task_id, + operation_type=ResourceOperation.APPLY.value + ) + if apply_status_code == FederatedSchedulingStatusCode.SUCCESS: + return True + else: + # rollback resource + rollback_party = [] + failed_party = [] + for dest_role in federated_response.keys(): + for dest_party_id in federated_response[dest_role].keys(): + retcode = federated_response[dest_role][dest_party_id]["code"] + if retcode == ReturnCode.Base.SUCCESS: + rollback_party.append({"role": dest_role, "party_id": [dest_party_id]}) + else: + failed_party.append({"role": dest_role, "party_id": [dest_party_id]}) + schedule_logger(job.f_job_id).info("task apply resource failed on {}, rollback {}".format(failed_party, + rollback_party)) + if rollback_party: + return_status_code, federated_response = FederatedScheduler.resource_for_task( + task_id=task.f_task_id, + roles=rollback_party, + operation_type=ResourceOperation.RETURN.value + ) + if return_status_code != FederatedSchedulingStatusCode.SUCCESS: + schedule_logger(job.f_job_id).info(f"task return resource failed:\n{federated_response}") + else: + schedule_logger(job.f_job_id).info("task no party should be rollback resource") + return False + + @classmethod + def collect_task_of_all_party(cls, job, task, set_status=None): + tasks_on_all_party = ScheduleJobSaver.query_task(task_id=task.f_task_id, task_version=task.f_task_version) + # tasks_status_on_all = set([task.f_status for task in tasks_on_all_party]) + # if not len(tasks_status_on_all) > 1 and TaskStatus.RUNNING not in tasks_status_on_all: + # return + status, federated_response = FederatedScheduler.collect_task(task_id=task.f_task_id) + if status != FederatedSchedulingStatusCode.SUCCESS: + schedule_logger(job.f_job_id).warning(f"collect task {task.f_task_id} {task.f_task_version} failed") + for _role in federated_response.keys(): + for _party_id, party_response in federated_response[_role].items(): + if party_response["code"] == ReturnCode.Base.SUCCESS: + schedule_logger(job.f_job_id).info( + f"collect party id {_party_id} task info: {party_response['data']}") + ScheduleJobSaver.update_task_status(task_info=party_response["data"]) + elif set_status: + tmp_task_info = { + "job_id": task.f_job_id, + "task_id": task.f_task_id, + "task_version": task.f_task_version, + "role": _role, + "party_id": _party_id, + "party_status": set_status + } + ScheduleJobSaver.update_task_status(task_info=tmp_task_info) + + @classmethod + def get_federated_task_status(cls, job_id, task_id, task_version): + tasks_on_all_party = ScheduleJobSaver.query_task(task_id=task_id, task_version=task_version) + if not tasks_on_all_party: + schedule_logger(job_id).error(f"task {task_id} {task_version} no found") + return TaskStatus.FAILED + tasks_party_status = [task.f_status for task in tasks_on_all_party] + status = cls.calculate_multi_party_task_status(tasks_party_status) + schedule_logger(job_id=job_id).info( + "task {} {} status is {}, calculate by task party status list: {}".format(task_id, task_version, status, + tasks_party_status)) + return status + + @classmethod + def calculate_multi_party_task_status(cls, tasks_party_status): + tmp_status_set = set(tasks_party_status) + if TaskStatus.PASS in tmp_status_set: + tmp_status_set.remove(TaskStatus.PASS) + tmp_status_set.add(TaskStatus.SUCCESS) + if len(tmp_status_set) == 1: + return tmp_status_set.pop() + else: + for status in sorted(InterruptStatus.status_list(), key=lambda s: StatusSet.get_level(status=s), + reverse=False): + if status in tmp_status_set: + return status + if TaskStatus.RUNNING in tmp_status_set: + return TaskStatus.RUNNING + if TaskStatus.SUCCESS in tmp_status_set: + return TaskStatus.RUNNING + raise Exception("Calculate task status failed: {}".format(tasks_party_status)) + + diff --git a/python/fate_flow/scheduler/task_scheduler.py b/python/fate_flow/scheduler/task_scheduler.py deleted file mode 100644 index 2b68242b6..000000000 --- a/python/fate_flow/scheduler/task_scheduler.py +++ /dev/null @@ -1,251 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_arch.common import FederatedCommunicationType -from fate_flow.entity import RetCode -from fate_flow.entity.run_status import StatusSet, TaskStatus, EndStatus, AutoRerunStatus, InterruptStatus -from fate_flow.entity.run_status import FederatedSchedulingStatusCode -from fate_flow.entity.run_status import SchedulingStatusCode -from fate_flow.entity import RunParameters -from fate_flow.utils import job_utils -from fate_flow.scheduler.federated_scheduler import FederatedScheduler -from fate_flow.operation.job_saver import JobSaver -from fate_flow.utils.log_utils import schedule_logger -from fate_flow.manager.resource_manager import ResourceManager -from fate_flow.controller.job_controller import JobController -from fate_flow.db.db_models import Job, Task -from fate_flow.entity.types import TaskCleanResourceType - - -class TaskScheduler(object): - @classmethod - def schedule(cls, job, dsl_parser, canceled=False): - schedule_logger(job.f_job_id).info("scheduling job tasks") - initiator_tasks_group = JobSaver.get_tasks_asc(job_id=job.f_job_id, role=job.f_role, party_id=job.f_party_id) - waiting_tasks = [] - auto_rerun_tasks = [] - job_interrupt = False - for initiator_task in initiator_tasks_group.values(): - if job.f_runtime_conf_on_party["job_parameters"]["federated_status_collect_type"] == FederatedCommunicationType.PULL: - # collect all parties task party status and store it in the database now - cls.collect_task_of_all_party(job=job, initiator_task=initiator_task) - else: - # all parties report task party status and store it in the initiator database when federated_status_collect_type is push - pass - # get all parties party task status and calculate - new_task_status = cls.get_federated_task_status(job_id=initiator_task.f_job_id, task_id=initiator_task.f_task_id, task_version=initiator_task.f_task_version) - task_interrupt = False - task_status_have_update = False - if new_task_status != initiator_task.f_status: - task_status_have_update = True - initiator_task.f_status = new_task_status - FederatedScheduler.sync_task_status(job=job, task=initiator_task) - if InterruptStatus.contains(new_task_status): - task_interrupt = True - job_interrupt = True - if initiator_task.f_status == TaskStatus.WAITING: - waiting_tasks.append(initiator_task) - elif task_status_have_update and EndStatus.contains(initiator_task.f_status) or task_interrupt: - command_body = {"is_asynchronous": True} - schedule_logger(initiator_task.f_job_id).info(f"stop task body: {command_body}, task status: {initiator_task.f_status}") - FederatedScheduler.stop_task(job=job, task=initiator_task, stop_status=initiator_task.f_status, command_body=command_body) - if not canceled and AutoRerunStatus.contains(initiator_task.f_status): - if initiator_task.f_auto_retries > 0: - auto_rerun_tasks.append(initiator_task) - schedule_logger(job.f_job_id).info(f"task {initiator_task.f_task_id} {initiator_task.f_status} will be retried") - else: - schedule_logger(job.f_job_id).info(f"task {initiator_task.f_task_id} {initiator_task.f_status} has no retry count") - - scheduling_status_code = SchedulingStatusCode.NO_NEXT - schedule_logger(job.f_job_id).info(f"canceled status {canceled}, job interrupt status {job_interrupt}") - if not canceled and not job_interrupt: - for waiting_task in waiting_tasks: - for component in dsl_parser.get_upstream_dependent_components(component_name=waiting_task.f_component_name): - dependent_task = initiator_tasks_group[ - JobSaver.task_key(task_id=job_utils.generate_task_id(job_id=job.f_job_id, component_name=component.get_name()), - role=job.f_role, - party_id=job.f_party_id - ) - ] - if dependent_task.f_status != TaskStatus.SUCCESS: - # can not start task - break - else: - # all upstream dependent tasks have been successful, can start this task - scheduling_status_code = SchedulingStatusCode.HAVE_NEXT - status_code = cls.start_task(job=job, task=waiting_task) - if status_code == SchedulingStatusCode.NO_RESOURCE: - # wait for the next round of scheduling - schedule_logger(job.f_job_id).info(f"task {waiting_task.f_task_id} can not apply resource, wait for the next round of scheduling") - break - elif status_code == SchedulingStatusCode.FAILED: - scheduling_status_code = SchedulingStatusCode.FAILED - waiting_task.f_status = StatusSet.FAILED - FederatedScheduler.sync_task_status(job, waiting_task) - break - else: - schedule_logger(job.f_job_id).info("have cancel signal, pass start job tasks") - schedule_logger(job.f_job_id).info("finish scheduling job tasks") - return scheduling_status_code, auto_rerun_tasks, initiator_tasks_group.values() - - @classmethod - def start_task(cls, job, task): - schedule_logger(task.f_job_id).info("try to start task {} {} on {} {}".format(task.f_task_id, task.f_task_version, task.f_role, task.f_party_id)) - apply_status = ResourceManager.apply_for_task_resource(task_info=task.to_human_model_dict(only_primary_with=["status"])) - if not apply_status: - return SchedulingStatusCode.NO_RESOURCE - task.f_status = TaskStatus.RUNNING - update_status = JobSaver.update_task_status(task_info=task.to_human_model_dict(only_primary_with=["status"])) - if not update_status: - # Another scheduler scheduling the task - schedule_logger(task.f_job_id).info("task {} {} start on another scheduler".format(task.f_task_id, task.f_task_version)) - # Rollback - task.f_status = TaskStatus.WAITING - ResourceManager.return_task_resource(task_info=task.to_human_model_dict(only_primary_with=["status"])) - return SchedulingStatusCode.PASS - schedule_logger(task.f_job_id).info("start task {} {} on {} {}".format(task.f_task_id, task.f_task_version, task.f_role, task.f_party_id)) - FederatedScheduler.sync_task_status(job=job, task=task) - status_code, response = FederatedScheduler.start_task(job=job, task=task) - if status_code == FederatedSchedulingStatusCode.SUCCESS: - return SchedulingStatusCode.SUCCESS - else: - return SchedulingStatusCode.FAILED - - @classmethod - def prepare_rerun_task(cls, job: Job, task: Task, dsl_parser, auto=False, force=False): - job_id = job.f_job_id - can_rerun = False - if force: - can_rerun = True - auto = False - schedule_logger(job_id).info(f"task {task.f_task_id} {task.f_task_version} with {task.f_status} was forced to rerun") - elif task.f_status in {TaskStatus.SUCCESS}: - schedule_logger(job_id).info(f"task {task.f_task_id} {task.f_task_version} is {task.f_status} and not force reruen, pass rerun") - elif auto and task.f_auto_retries < 1: - schedule_logger(job_id).info(f"task {task.f_task_id} has no retry count, pass rerun") - else: - can_rerun = True - if can_rerun: - if task.f_status != TaskStatus.WAITING: - cls.create_new_version_task(job=job, - task=task, - dsl_parser=dsl_parser, - auto=auto) - return can_rerun - - @classmethod - def create_new_version_task(cls, job, task, dsl_parser, auto): - # stop old version task - FederatedScheduler.stop_task(job=job, task=task, stop_status=TaskStatus.CANCELED) - FederatedScheduler.clean_task(job=job, task=task, content_type=TaskCleanResourceType.METRICS) - # create new version task - task.f_task_version = task.f_task_version + 1 - if auto: - task.f_auto_retries = task.f_auto_retries - 1 - task.f_run_pid = None - task.f_run_ip = None - # todo: FederatedScheduler.create_task and JobController.initialize_tasks will create task twice - status_code, response = FederatedScheduler.create_task(job=job, task=task) - if status_code != FederatedSchedulingStatusCode.SUCCESS: - raise Exception(f"create {task.f_task_id} new version failed") - # create the task holder in db to record information of all participants in the initiator for scheduling - for _role in response: - for _party_id in response[_role]: - if _role == job.f_initiator_role and _party_id == job.f_initiator_party_id: - continue - JobController.initialize_tasks(job_id=job.f_job_id, - role=_role, - party_id=_party_id, - run_on_this_party=False, - initiator_role=job.f_initiator_role, - initiator_party_id=job.f_initiator_party_id, - job_parameters=RunParameters(**job.f_runtime_conf_on_party["job_parameters"]), - dsl_parser=dsl_parser, - components=[task.f_component_name], - task_version=task.f_task_version, - auto_retries=task.f_auto_retries, - runtime_conf=job.f_runtime_conf) - schedule_logger(job.f_job_id).info(f"create task {task.f_task_id} new version {task.f_task_version} successfully") - - @classmethod - def collect_task_of_all_party(cls, job, initiator_task, set_status=None): - tasks_on_all_party = JobSaver.query_task(task_id=initiator_task.f_task_id, task_version=initiator_task.f_task_version) - tasks_status_on_all = set([task.f_status for task in tasks_on_all_party]) - if not len(tasks_status_on_all) > 1 and TaskStatus.RUNNING not in tasks_status_on_all: - return - status, federated_response = FederatedScheduler.collect_task(job=job, task=initiator_task) - if status != FederatedSchedulingStatusCode.SUCCESS: - schedule_logger(job.f_job_id).warning(f"collect task {initiator_task.f_task_id} {initiator_task.f_task_version} on {initiator_task.f_role} {initiator_task.f_party_id} failed") - for _role in federated_response.keys(): - for _party_id, party_response in federated_response[_role].items(): - if party_response["retcode"] == RetCode.SUCCESS: - JobSaver.update_task_status(task_info=party_response["data"]) - JobSaver.update_task(task_info=party_response["data"]) - elif party_response["retcode"] == RetCode.FEDERATED_ERROR and set_status: - tmp_task_info = { - "job_id": initiator_task.f_job_id, - "task_id": initiator_task.f_task_id, - "task_version": initiator_task.f_task_version, - "role": _role, - "party_id": _party_id, - "party_status": TaskStatus.RUNNING - } - JobSaver.update_task_status(task_info=tmp_task_info) - tmp_task_info["party_status"] = set_status - JobSaver.update_task_status(task_info=tmp_task_info) - - @classmethod - def get_federated_task_status(cls, job_id, task_id, task_version): - tasks_on_all_party = JobSaver.query_task(task_id=task_id, task_version=task_version) - status_flag = 0 - # idmapping role status can only be ignored if all non-idmapping roles success - for task in tasks_on_all_party: - if 'idmapping' not in task.f_role and task.f_party_status != TaskStatus.SUCCESS: - status_flag = 1 - break - if status_flag: - tasks_party_status = [task.f_party_status for task in tasks_on_all_party] - else: - tasks_party_status = [task.f_party_status for task in tasks_on_all_party if 'idmapping' not in task.f_role] - status = cls.calculate_multi_party_task_status(tasks_party_status) - schedule_logger(job_id=job_id).info("task {} {} status is {}, calculate by task party status list: {}".format(task_id, task_version, status, tasks_party_status)) - return status - - @classmethod - def calculate_multi_party_task_status(cls, tasks_party_status): - # 1. all waiting - # 2. have interrupt status, should be interrupted - # 3. have running - # 4. waiting + success/pass - # 5. all the same end status - tmp_status_set = set(tasks_party_status) - if TaskStatus.PASS in tmp_status_set: - tmp_status_set.remove(TaskStatus.PASS) - tmp_status_set.add(TaskStatus.SUCCESS) - if len(tmp_status_set) == 1: - # 1 and 5 - return tmp_status_set.pop() - else: - # 2 - for status in sorted(InterruptStatus.status_list(), key=lambda s: StatusSet.get_level(status=s), reverse=True): - if status in tmp_status_set: - return status - # 3 - if TaskStatus.RUNNING in tmp_status_set: - return TaskStatus.RUNNING - # 4 - if TaskStatus.SUCCESS in tmp_status_set: - return TaskStatus.RUNNING - raise Exception("Calculate task status failed: {}".format(tasks_party_status)) diff --git a/python/fate_flow/scheduler/tests/dsl_parser/cache_dsl.json b/python/fate_flow/scheduler/tests/dsl_parser/cache_dsl.json deleted file mode 100644 index a029e2857..000000000 --- a/python/fate_flow/scheduler/tests/dsl_parser/cache_dsl.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "data" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "cache": [ - "cache" - ] - } - }, - "intersection_1": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.data" - ] - }, - "cache": ["intersection_0.cache"] - }, - "output": { - "data": [ - "data" - ] - } - } - } -} diff --git a/python/fate_flow/scheduler/tests/dsl_parser/component_parameters.json b/python/fate_flow/scheduler/tests/dsl_parser/component_parameters.json deleted file mode 100644 index cccfeb51c..000000000 --- a/python/fate_flow/scheduler/tests/dsl_parser/component_parameters.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "dsl_version": "2", - "initiator": { - "role": "guest", - "party_id": 9999 - }, - "role": { - "guest": [ - 9999 - ], - "host": [ - 10000 - ] - }, - "job_parameters": { - "common": { - "job_type": "train", - "work_mode": 0, - "backend": 0, - "computing_engine": "STANDALONE", - "federation_engine": "STANDALONE", - "storage_engine": "STANDALONE", - "engines_address": {}, - "federated_mode": "SINGLE", - "task_cores": 2, - "task_parallelism": 1, - "computing_partitions": 8, - "federated_status_collect_type": "PUSH", - "model_id": "arbiter-10000#guest-9999#host-10000#model", - "model_version": "202108062006408966120", - "auto_retries": 1, - "auto_retry_delay": 1, - "component_provider": "fate_federated_algorithm", - "component_version": "1.7.0", - "eggroll_run": {}, - "spark_run": {}, - "rabbitmq_run": {}, - "pulsar_run": {}, - "adaptation_parameters": { - "task_nodes": 1, - "task_cores_per_node": 2, - "task_memory_per_node": 0, - "request_task_cores": 2, - "if_initiator_baseline": true - } - } - }, - "config": "fate/python/fate_flow/examples/simple_hetero_lr_job_conf.json", - "dsl": "examples/simple_dsl.json", - "function": "submit_job", - "local": { - "role": "guest", - "party_id": 9999 - }, - "module": "HeteroFeatureBinning", - "CodePath": "HeteroFeatureBinningGuest", - "ComponentParam": { - "method": "quantile", - "compress_thres": 10000, - "head_size": 10000, - "error": 0.0001, - "adjustment_factor": 0.5, - "bin_num": 10, - "bin_indexes": -1, - "bin_names": null, - "category_indexes": null, - "category_names": null, - "transform_param": { - "transform_cols": -1, - "transform_names": null, - "transform_type": "bin_num" - }, - "need_run": true, - "skip_static": false, - "local_only": false, - "optimal_binning_param": { - "init_bucket_method": "quantile", - "metric_method": "iv", - "max_bin": null, - "mixture": true, - "max_bin_pct": 1.0, - "min_bin_pct": 0.05, - "init_bin_nums": 1000, - "adjustment_factor": null - }, - "encrypt_param": { - "method": "Paillier", - "key_length": 1024 - } - } -} \ No newline at end of file diff --git a/python/fate_flow/scheduler/tests/dsl_parser/test_dsl_parser_cache.py b/python/fate_flow/scheduler/tests/dsl_parser/test_dsl_parser_cache.py deleted file mode 100644 index 1bbf99a43..000000000 --- a/python/fate_flow/scheduler/tests/dsl_parser/test_dsl_parser_cache.py +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import pprint -import sys -from fate_flow.scheduler import dsl_parser - -dsl_path = sys.argv[1] -conf_path = sys.argv[2] -provider_path = sys.argv[3] - -with open(dsl_path, "r") as fin: - dsl = json.loads(fin.read()) - -with open(conf_path, "r") as fin: - conf = json.loads(fin.read()) - -with open(provider_path, "r") as fin: - provider_detail = json.loads(fin.read()) - - -dsl_parser_v2 = dsl_parser.DSLParserV2() -dsl_parser_v2.run(dsl=dsl, - runtime_conf=conf, - mode="train") - -pprint.pprint(dsl_parser_v2.get_job_parameters()) -print ("\n\n\n") -pprint.pprint(dsl_parser_v2.get_job_providers(provider_detail=provider_detail, - local_role="arbiter", - local_party_id=9999)) -print ("\n\n\n") -pprint.pprint(dsl_parser_v2.get_dependency()) -print ("\n\n\n") - -job_providers = dsl_parser_v2.get_job_providers(provider_detail=provider_detail, - local_role="arbiter", - local_party_id=9999) -component_parameters = dict() -for component in job_providers.keys(): - provider_info = job_providers[component]["provider"] - provider_name = provider_info["name"] - provider_version = provider_info["version"] - - parameter = dsl_parser_v2.parse_component_parameters(component, - provider_detail, - provider_name, - provider_version, - local_role="arbiter", - local_party_id=9999) - - component_parameters[component] = parameter - pprint.pprint (parameter) - -pprint.pprint(dsl_parser_v2.get_dependency_with_parameters(component_parameters)) -print ("\n\n\n") - - -pprint.pprint(dsl_parser_v2.deploy_component(["reader_0", "data_transform_0", "intersection_0"], dsl)) -print ("\n\n\n") - -pprint.pprint(dsl_parser_v2.deploy_component(["reader_0", "data_transform_0", "intersection_1"], dsl)) -print ("\n\n\n") - -pprint.pprint(dsl_parser_v2.deploy_component(["reader_0", "data_transform_0", "intersection_0", "intersection_1"], dsl)) -print ("\n\n\n") diff --git a/python/fate_flow/scheduler/tests/dsl_parser/test_dsl_parser_v1.py b/python/fate_flow/scheduler/tests/dsl_parser/test_dsl_parser_v1.py deleted file mode 100644 index f68469a78..000000000 --- a/python/fate_flow/scheduler/tests/dsl_parser/test_dsl_parser_v1.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import pprint -import sys -from fate_flow.scheduler import dsl_parser - -dsl_path_v1 = sys.argv[1] -conf_path_v1 = sys.argv[2] -provider_path = sys.argv[3] - -"""test dsl v2""" -with open(dsl_path_v1, "r") as fin: - dsl_v1 = json.loads(fin.read()) - -with open(conf_path_v1, "r") as fin: - conf_v1 = json.loads(fin.read()) - -with open(provider_path, "r") as fin: - provider_detail = json.loads(fin.read()) - - -dsl_parser_v1 = dsl_parser.DSLParserV1() -dsl_v2, warning_msg = dsl_parser_v1.convert_dsl_v1_to_v2(dsl_v1) - -pprint.pprint(dsl_v2) -print (warning_msg) -exit(0) -components = dsl_parser_v1.get_components_light_weight(dsl_v2) -for cpn in components: - print (cpn.get_name()) - print (cpn.get_module()) - -pprint.pprint(dsl_parser_v1.get_job_parameters(conf_v1)) - -job_providers = dsl_parser_v1.get_job_providers(dsl=dsl_v2, provider_detail=provider_detail) -pprint.pprint(job_providers) -print("\n\n\n") - -cpn_role_parameters = dict() -for cpn in components: - cpn_name = cpn.get_name() - role_params = dsl_parser_v1.parse_component_role_parameters(component=cpn_name, - dsl=dsl_v2, - runtime_conf=conf_v1, - provider_detail=provider_detail, - provider_name=job_providers[cpn_name]["provider"]["name"], - provider_version=job_providers[cpn_name]["provider"]["version"]) - - print (cpn_name) - pprint.pprint(role_params) - print ("\n") - - cpn_role_parameters[cpn_name] = role_params - -print ("\n\n\n") - -conf_v2 = dsl_parser_v1.convert_conf_v1_to_v2(conf_v1, cpn_role_parameters) -pprint.pprint(conf_v2) diff --git a/python/fate_flow/scheduler/tests/dsl_parser/test_dsl_parser_v2.py b/python/fate_flow/scheduler/tests/dsl_parser/test_dsl_parser_v2.py deleted file mode 100644 index 80370bcbc..000000000 --- a/python/fate_flow/scheduler/tests/dsl_parser/test_dsl_parser_v2.py +++ /dev/null @@ -1,159 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import copy -import json -import pprint -import sys -from fate_flow.scheduler import dsl_parser - -dsl_path_v2 = sys.argv[1] -conf_path_v2 = sys.argv[2] -provider_path = sys.argv[3] - -"""test dsl v2""" -with open(dsl_path_v2, "r") as fin: - dsl_v2 = json.loads(fin.read()) - -with open(conf_path_v2, "r") as fin: - conf_v2 = json.loads(fin.read()) - -with open(provider_path, "r") as fin: - provider_detail = json.loads(fin.read()) - -dsl_parser_v2 = dsl_parser.DSLParserV2() -dsl_parser_v2.run(dsl=dsl_v2, - runtime_conf=conf_v2, - mode="train") - -pprint.pprint(dsl_parser_v2.get_job_parameters()) -print("\n\n\n") -pprint.pprint(dsl_parser_v2.get_job_providers(provider_detail=provider_detail, conf=conf_v2, - local_role="guest", local_party_id=10000)) - -exit(0) -print("\n\n\n") -pprint.pprint(dsl_parser_v2.get_dependency()) -pprint.pprint(dsl_parser_v2.get_dsl_hierarchical_structure()) -print("\n\n\n") - -job_providers = dsl_parser_v2.get_job_providers(provider_detail=provider_detail) - -component_parameters = dict() -for component in job_providers.keys(): - provider_info = job_providers[component]["provider"] - provider_name = provider_info["name"] - provider_version = provider_info["version"] - - parameter = dsl_parser_v2.parse_component_parameters(component, - provider_detail, - provider_name, - provider_version, - local_role="guest", - local_party_id=10000) - - user_parameter = dsl_parser_v2.parse_user_specified_component_parameters(component, - provider_detail, - provider_name, - provider_version, - local_role="guest", - local_party_id=10000) - - component_parameters[component] = parameter - # pprint.pprint(component) - # pprint.pprint(parameter) - - pprint.pprint(user_parameter) - print("\n\n\n") - -evaluation_paramters = {'CodePath': 'Evaluation', - 'ComponentParam': {'eval_type': 'multi', "test": "test_keyword"}, - 'dsl_version': 2, - 'initiator': {'party_id': 10000, 'role': 'guest'}, - 'job_parameters': {'common': {'backend': 0, - 'job_type': 'train', - 'work_mode': 1}}, - 'local': {'party_id': 10000, 'role': 'guest'}, - 'module': 'Evaluation', - 'role': {'arbiter': [9999], 'guest': [10000], 'host': [9999]}} - -provider_info = job_providers["evaluation_0"]["provider"] -provider_name = provider_info["name"] -provider_version = provider_info["version"] - -new_evaluation_parameter = dsl_parser_v2.parse_component_parameters("evaluation_0", - provider_detail, - provider_name, - provider_version, - local_role="guest", - local_party_id=10000, - previous_parameters={"evaluation_0": evaluation_paramters}) - -pprint.pprint(new_evaluation_parameter) -pprint.pprint(dsl_parser_v2.get_dependency_with_parameters(component_parameters)) -print("\n\n\n") - -print(dsl_parser_v2.get_dsl_hierarchical_structure()) -print(dsl_parser_v2.get_dsl_hierarchical_structure()[0]["reader_0"].get_component_provider()) -print("\n\n\n") - -pprint.pprint(dsl_parser_v2.deploy_component(["reader_0", "data_transform_0"], dsl_v2)) -print("\n\n\n") - -module_object_name_mapping = dict() -for component in job_providers.keys(): - module = dsl_v2["components"][component]["module"] - provider_info = job_providers[component]["provider"] - provider_name = provider_info["name"] - provider_version = provider_info["version"] - module_object = dsl_parser_v2.get_module_object_name(module, "guest", provider_detail, - provider_name, provider_version) - - module_object_name_mapping[component] = module_object - -pprint.pprint(dsl_parser_v2.get_predict_dsl(dsl_v2, module_object_name_mapping)) -print(dsl_parser_v2.get_downstream_dependent_components("data_transform_0")) -print(dsl_parser_v2.get_upstream_dependent_components("data_transform_0")) - - -dsl = copy.deepcopy(dsl_v2) -del dsl["components"]["reader_0"] -del dsl["components"]["data_transform_0"] -del dsl["components"]["hetero_feature_selection_0"] - -print(dsl_parser_v2.check_input_existence(dsl)) -print("\n\n\n") - -conf_v2["component_parameters"]["common"]["evaluation_0"]["test"] = "test" -provider_info = job_providers["evaluation_0"]["provider"] -provider_name = provider_info["name"] -provider_version = provider_info["version"] - -try: - dsl_parser_v2.validate_component_param(component="evaluation_0", - module="Evaluation", - runtime_conf=conf_v2, - provider_name=provider_name, - provider_version=provider_version, - provider_detail=provider_detail, - local_role="guest", - local_party_id=10000) -except Exception as e: - print (e) - - - - diff --git a/python/fate_flow/scheduler/tests/dsl_parser/v1_conf.json b/python/fate_flow/scheduler/tests/dsl_parser/v1_conf.json deleted file mode 100644 index 68f9d651f..000000000 --- a/python/fate_flow/scheduler/tests/dsl_parser/v1_conf.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "initiator": { - "role": "guest", - "party_id": 10000 - }, - "job_parameters": { - "work_mode": 0 - }, - "role": { - "guest": [ - 10000 - ], - "host": [ - 10000 - ], - "arbiter": [ - 10000 - ] - }, - "role_parameters": { - "guest": { - "args": { - "data": { - "train_data": [ - { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - ], - "eval_data": [ - { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - ] - } - }, - "data_transform_0": { - "with_label": [ - true - ], - "label_name": [ - "y" - ], - "label_type": [ - "int" - ], - "output_format": [ - "dense" - ], - "missing_fill": [ - true - ], - "outlier_replace": [ - true - ] - }, - "evaluation_0": { - "eval_type": [ - "binary" - ], - "pos_label": [ - 1 - ] - } - }, - "host": { - "args": { - "data": { - "train_data": [ - { - "name": "breast_hetero_host", - "namespace": "experiment" - } - ], - "eval_data": [ - { - "name": "breast_hetero_host", - "namespace": "experiment" - } - ] - } - }, - "data_transform_0": { - "with_label": [ - false - ], - "output_format": [ - "dense" - ], - "outlier_replace": [ - true - ] - }, - "evaluation_0": { - "need_run": [ - false - ] - } - } - }, - "algorithm_parameters": { - "hetero_lr_0": { - "penalty": "L2", - "optimizer": "rmsprop", - "tol": 0.0001, - "alpha": 0.01, - "max_iter": 30, - "early_stop": "diff", - "batch_size": 233, - "learning_rate": 0.15, - "init_param": { - "init_method": "zeros" - }, - "sqn_param": { - "update_interval_L": 3, - "memory_M": 5, - "sample_size": 5000, - "random_seed": null - }, - "cv_param": { - "n_splits": 5, - "shuffle": true, - "random_seed": 103, - "need_cv": false - } - }, - "intersect_0": { - "intersect_method": "rsa", - "sync_intersect_ids": true, - "only_output_key": false - } - } -} diff --git a/python/fate_flow/scheduler/tests/dsl_parser/v1_dsl.json b/python/fate_flow/scheduler/tests/dsl_parser/v1_dsl.json deleted file mode 100644 index 3f6c34aa8..000000000 --- a/python/fate_flow/scheduler/tests/dsl_parser/v1_dsl.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "components" : { - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "args.train_data" - ] - } - }, - "output": { - "data": ["train"], - "model": ["data_transform"] - } - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.train" - ] - } - }, - "output": { - "data": ["train"] - } - }, - "hetero_lr_0": { - "module": "HeteroLR", - "input": { - "data": { - "train_data": ["intersection_0.train"] - } - }, - "output": { - "data": ["train"], - "model": ["hetero_lr"] - } - }, - "evaluation_0": { - "module": "Evaluation", - "input": { - "data": { - "data": ["hetero_lr_0.train"] - } - } - } - } -} diff --git a/python/fate_flow/scheduler/tests/dsl_parser/v2_conf.json b/python/fate_flow/scheduler/tests/dsl_parser/v2_conf.json deleted file mode 100644 index 331d3997b..000000000 --- a/python/fate_flow/scheduler/tests/dsl_parser/v2_conf.json +++ /dev/null @@ -1,143 +0,0 @@ -{ - "dsl_version": 2, - "initiator": { - "role": "guest", - "party_id": 10000 - }, - "role": { - "guest": [ - 10000 - ], - "host": [ - 9999, 9998, 9997 - ], - "arbiter": [ - 9999 - ] - }, - "job_parameters": { - "common": { - "job_type": "train", - "backend": 0, - "work_mode": 1 - } - }, - "component_parameters": { - "common": { - "feature_scale_0": { - "method": "standard_scale", - "need_run": true - }, - "hetero_feature_binning_0": { - "method": "quantile", - "compress_thres": 10000, - "head_size": 10000, - "error": 0.001, - "adjustment_factor": 0.5, - "bin_num": 10, - "bin_indexes": -1, - "local_only": false, - "transform_param": { - "transform_cols": -1, - "transform_type": "bin_num" - }, - "need_run": true - }, - "hetero_feature_selection_0": { - "select_col_indexes": -1, - "filter_methods": [ - "manually", - "iv_value_thres", - "iv_percentile" - ], - "iv_value_param": { - "value_threshold": 1.0 - }, - "iv_percentile_param": { - "percentile_threshold": 0.9 - }, - "manually_param": { - "filter_out_indexes": null - }, - "need_run": true - }, - "one_hot_encoder_0": { - "transform_col_indexes": -1, - "transform_col_names": [], - "need_run": true - }, - "hetero_lr_0": { - "penalty": "L2", - "tol": 1e-05, - "alpha": 0.01, - "optimizer": "rmsprop", - "batch_size": -1, - "learning_rate": 0.15, - "init_param": { - "init_method": "random_uniform" - }, - "max_iter": 10, - "early_stop": "diff", - "cv_param": { - "n_splits": 5, - "shuffle": false, - "random_seed": 103, - "need_cv": false - } - }, - "evaluation_0": { - "eval_type": "binary" - } - }, - "role": { - "host": { - "0": { - "data_transform_0": { - "with_label": false - }, - "reader_0": { - "table": { - "name": "breast_hetero_host", - "namespace": "experiment" - } - } - } - }, - "guest": { - "0": { - "data_transform_0": { - "with_label": true, - "output_format": "dense" - }, - "reader_0": { - "table": { - "name": "breast_hetero_guest", - "namespace": "experiment" - } - } - } - } - } - }, - "provider": { - "common": { - "hetero_feature_binning_0": "fate@1.8.0" - }, - "role": { - "guest": { - "0": { - "data_transform_0": "fate@1.8.0" - } - }, - "host": { - "0": { - "data_transform_0": "fate@1.7.3", - "one_hot_encoder_0": "fate@1.7.3" - }, - "1|2": { - "data_transform_0": "fate@1.7.4" - } - } - } - } -} diff --git a/python/fate_flow/scheduler/tests/dsl_parser/v2_dsl.json b/python/fate_flow/scheduler/tests/dsl_parser/v2_dsl.json deleted file mode 100644 index 8e675d7f7..000000000 --- a/python/fate_flow/scheduler/tests/dsl_parser/v2_dsl.json +++ /dev/null @@ -1,174 +0,0 @@ -{ - "components": { - "reader_0": { - "module": "Reader", - "output": { - "data": [ - "data" - ] - } - }, - "data_transform_0": { - "module": "DataTransform", - "input": { - "data": { - "data": [ - "reader_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - }, - "provider": "fate@1.8.0" - }, - "intersection_0": { - "module": "Intersection", - "input": { - "data": { - "data": [ - "data_transform_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ] - }, - "provider": "fate" - }, - "feature_scale_0": { - "module": "FeatureScale", - "input": { - "data": { - "data": [ - "intersection_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "hetero_feature_binning_0": { - "module": "HeteroFeatureBinning", - "input": { - "data": { - "data": [ - "feature_scale_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "hetero_feature_selection_0": { - "module": "HeteroFeatureSelection", - "input": { - "data": { - "data": [ - "hetero_feature_binning_0.data" - ] - }, - "isometric_model": [ - "hetero_feature_binning_0.model" - ] - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "one_hot_encoder_0": { - "module": "OneHotEncoder", - "input": { - "data": { - "data": [ - "hetero_feature_selection_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "hetero_lr_0": { - "module": "HeteroLR", - "input": { - "data": { - "train_data": [ - "one_hot_encoder_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "hetero_lr_1": { - "module": "HeteroLR", - "input": { - "data": { - "train_data": [ - "one_hot_encoder_0.data" - ] - }, - "model": ["hetero_lr_0.model"] - }, - "output": { - "data": [ - "data" - ], - "model": [ - "model" - ] - } - }, - "evaluation_0": { - "module": "Evaluation", - "input": { - "data": { - "data": [ - "hetero_lr_0.data" - ] - } - }, - "output": { - "data": [ - "data" - ] - } - } - } -} diff --git a/python/fate_flow/scheduling_apps/client/control_client.py b/python/fate_flow/scheduling_apps/client/control_client.py deleted file mode 100644 index e6743af74..000000000 --- a/python/fate_flow/scheduling_apps/client/control_client.py +++ /dev/null @@ -1,54 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_flow.utils.log_utils import getLogger -from fate_flow.utils import api_utils - -LOGGER = getLogger() - - -class ControllerClient(object): - @classmethod - def update_job(cls, job_info): - LOGGER.info(f"request update job {job_info['job_id']} on {job_info['role']} {job_info['party_id']}: {job_info}") - response = api_utils.local_api( - job_id=job_info["job_id"], - method='POST', - endpoint='/party/{}/{}/{}/update'.format( - job_info["job_id"], - job_info["role"], - job_info["party_id"] - ), - json_body=job_info) - return response - - @classmethod - def report_task(cls, task_info): - LOGGER.info("request update job {} task {} {} on {} {}".format(task_info["job_id"], task_info["task_id"], - task_info["task_version"], task_info["role"], - task_info["party_id"])) - response = api_utils.local_api( - job_id=task_info["job_id"], - method='POST', - endpoint='/party/{}/{}/{}/{}/{}/{}/report'.format( - task_info["job_id"], - task_info["component_name"], - task_info["task_id"], - task_info["task_version"], - task_info["role"], - task_info["party_id"] - ), - json_body=task_info) - return response diff --git a/python/fate_flow/scheduling_apps/client/operation_client.py b/python/fate_flow/scheduling_apps/client/operation_client.py deleted file mode 100644 index 0d25cd995..000000000 --- a/python/fate_flow/scheduling_apps/client/operation_client.py +++ /dev/null @@ -1,54 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_flow.utils.log_utils import getLogger -from fate_flow.utils import api_utils - -LOGGER = getLogger() - - -class OperationClient(object): - @classmethod - def get_job_conf(cls, job_id, role, party_id, component_name=None, task_id=None, task_version=None): - json_body = { - "job_id": job_id, - "role": role, - "party_id": party_id, - } - if component_name is not None and task_id is not None and task_version is not None: - json_body.update({ - "component_name": component_name, - "task_id": task_id, - "task_version": task_version, - }) - - response = api_utils.local_api( - job_id=job_id, - method='POST', - endpoint='/operation/job_config/get', - json_body=json_body, - ) - return response.get("data") - - - @classmethod - def load_json_conf(cls, job_id, config_path): - response = api_utils.local_api( - job_id=job_id, - method='POST', - endpoint='/operation/json_conf/load'.format( - ), - json_body={"config_path": config_path}) - return response.get("data") diff --git a/python/fate_flow/scheduling_apps/client/tracker_client.py b/python/fate_flow/scheduling_apps/client/tracker_client.py deleted file mode 100644 index 2aa73cb54..000000000 --- a/python/fate_flow/scheduling_apps/client/tracker_client.py +++ /dev/null @@ -1,300 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import base64 -import typing -from typing import List - -from fate_arch import storage -from fate_arch.abc import AddressABC -from fate_flow.utils.log_utils import getLogger -from fate_flow.entity import RunParameters -from fate_arch.common.base_utils import serialize_b64, deserialize_b64 -from fate_flow.entity import RetCode -from fate_flow.entity import Metric, MetricMeta -from fate_flow.operation.job_tracker import Tracker -from fate_flow.utils import api_utils - -LOGGER = getLogger() - - -class TrackerClient(object): - def __init__(self, job_id: str, role: str, party_id: int, - model_id: str = None, - model_version: str = None, - component_name: str = None, - component_module_name: str = None, - task_id: str = None, - task_version: int = None, - job_parameters: RunParameters = None - ): - self.job_id = job_id - self.role = role - self.party_id = party_id - self.model_id = model_id - self.model_version = model_version - self.component_name = component_name if component_name else 'pipeline' - self.module_name = component_module_name if component_module_name else 'Pipeline' - self.task_id = task_id - self.task_version = task_version - self.job_parameters = job_parameters - self.job_tracker = Tracker(job_id=job_id, role=role, party_id=party_id, component_name=component_name, - task_id=task_id, - task_version=task_version, - model_id=model_id, - model_version=model_version, - job_parameters=job_parameters) - - def log_job_metric_data(self, metric_namespace: str, metric_name: str, metrics: List[typing.Union[Metric, dict]]): - self.log_metric_data_common(metric_namespace=metric_namespace, metric_name=metric_name, metrics=metrics, - job_level=True) - - def log_metric_data(self, metric_namespace: str, metric_name: str, metrics: List[typing.Union[Metric, dict]]): - self.log_metric_data_common(metric_namespace=metric_namespace, metric_name=metric_name, metrics=metrics, - job_level=False) - - def log_metric_data_common(self, metric_namespace: str, metric_name: str, metrics: List[typing.Union[Metric, dict]], job_level=False): - LOGGER.info("Request save job {} task {} {} on {} {} metric {} {} data".format(self.job_id, - self.task_id, - self.task_version, - self.role, - self.party_id, - metric_namespace, - metric_name)) - request_body = {} - request_body['metric_namespace'] = metric_namespace - request_body['metric_name'] = metric_name - request_body['metrics'] = [serialize_b64(metric if isinstance(metric, Metric) else Metric.from_dict(metric), to_str=True) for metric in metrics] - request_body['job_level'] = job_level - response = api_utils.local_api(job_id=self.job_id, - method='POST', - endpoint='/tracker/{}/{}/{}/{}/{}/{}/metric_data/save'.format( - self.job_id, - self.component_name, - self.task_id, - self.task_version, - self.role, - self.party_id), - json_body=request_body) - if response['retcode'] != RetCode.SUCCESS: - raise Exception(f"log metric(namespace: {metric_namespace}, name: {metric_name}) data error, response code: {response['retcode']}, msg: {response['retmsg']}") - - def set_job_metric_meta(self, metric_namespace: str, metric_name: str, metric_meta: typing.Union[MetricMeta, dict]): - self.set_metric_meta_common(metric_namespace=metric_namespace, metric_name=metric_name, metric_meta=metric_meta, - job_level=True) - - def set_metric_meta(self, metric_namespace: str, metric_name: str, metric_meta: typing.Union[MetricMeta, dict]): - self.set_metric_meta_common(metric_namespace=metric_namespace, metric_name=metric_name, metric_meta=metric_meta, - job_level=False) - - def set_metric_meta_common(self, metric_namespace: str, metric_name: str, metric_meta: typing.Union[MetricMeta, dict], job_level=False): - LOGGER.info("Request save job {} task {} {} on {} {} metric {} {} meta".format(self.job_id, - self.task_id, - self.task_version, - self.role, - self.party_id, - metric_namespace, - metric_name)) - request_body = dict() - request_body['metric_namespace'] = metric_namespace - request_body['metric_name'] = metric_name - request_body['metric_meta'] = serialize_b64(metric_meta if isinstance(metric_meta, MetricMeta) else MetricMeta.from_dict(metric_meta), to_str=True) - request_body['job_level'] = job_level - response = api_utils.local_api(job_id=self.job_id, - method='POST', - endpoint='/tracker/{}/{}/{}/{}/{}/{}/metric_meta/save'.format( - self.job_id, - self.component_name, - self.task_id, - self.task_version, - self.role, - self.party_id), - json_body=request_body) - if response['retcode'] != RetCode.SUCCESS: - raise Exception(f"log metric(namespace: {metric_namespace}, name: {metric_name}) meta error, response code: {response['retcode']}, msg: {response['retmsg']}") - - def create_table_meta(self, table_meta): - request_body = dict() - for k, v in table_meta.to_dict().items(): - if k == "part_of_data": - request_body[k] = serialize_b64(v, to_str=True) - elif k == "schema": - request_body[k] = serialize_b64(v, to_str=True) - elif issubclass(type(v), AddressABC): - request_body[k] = v.__dict__ - else: - request_body[k] = v - response = api_utils.local_api(job_id=self.job_id, - method='POST', - endpoint='/tracker/{}/{}/{}/{}/{}/{}/table_meta/create'.format( - self.job_id, - self.component_name, - self.task_id, - self.task_version, - self.role, - self.party_id), - json_body=request_body) - if response['retcode'] != RetCode.SUCCESS: - raise Exception(f"create table meta failed:{response['retmsg']}") - - def get_table_meta(self, table_name, table_namespace): - request_body = {"table_name": table_name, "namespace": table_namespace} - response = api_utils.local_api(job_id=self.job_id, - method='POST', - endpoint='/tracker/{}/{}/{}/{}/{}/{}/table_meta/get'.format( - self.job_id, - self.component_name, - self.task_id, - self.task_version, - self.role, - self.party_id), - json_body=request_body) - if response['retcode'] != RetCode.SUCCESS: - raise Exception(f"create table meta failed:{response['retmsg']}") - else: - data_table_meta = storage.StorageTableMeta(name=table_name, - namespace=table_namespace, new=True) - data_table_meta.set_metas(**response["data"]) - data_table_meta.address = storage.StorageTableMeta.create_address(storage_engine=response["data"].get("engine"), - address_dict=response["data"].get("address")) - data_table_meta.part_of_data = deserialize_b64(data_table_meta.part_of_data) - data_table_meta.schema = deserialize_b64(data_table_meta.schema) - return data_table_meta - - def save_component_output_model(self, model_buffers: dict, model_alias: str, user_specified_run_parameters: dict = None): - component_model = self.job_tracker.pipelined_model.create_component_model(component_name=self.component_name, - component_module_name=self.module_name, - model_alias=model_alias, - model_buffers=model_buffers, - user_specified_run_parameters=user_specified_run_parameters) - json_body = {"model_id": self.model_id, "model_version": self.model_version, "component_model": component_model} - response = api_utils.local_api(job_id=self.job_id, - method='POST', - endpoint='/tracker/{}/{}/{}/{}/{}/{}/model/save'.format( - self.job_id, - self.component_name, - self.task_id, - self.task_version, - self.role, - self.party_id), - json_body=json_body) - if response['retcode'] != RetCode.SUCCESS: - raise Exception(f"save component output model failed:{response['retmsg']}") - - def read_component_output_model(self, search_model_alias): - json_body = {"search_model_alias": search_model_alias, "model_id": self.model_id, "model_version": self.model_version} - response = api_utils.local_api(job_id=self.job_id, - method='POST', - endpoint='/tracker/{}/{}/{}/{}/{}/{}/model/get'.format( - self.job_id, - self.component_name, - self.task_id, - self.task_version, - self.role, - self.party_id), - json_body=json_body) - if response['retcode'] != RetCode.SUCCESS: - raise Exception(f"get output model failed:{response['retmsg']}") - else: - model_buffers = {} - for model_name, v in response['data'].items(): - model_buffers[model_name] = (v[0], base64.b64decode(v[1].encode())) - return model_buffers - - def get_model_run_parameters(self): - json_body = {"model_id": self.model_id, "model_version": self.model_version} - response = api_utils.local_api(job_id=self.job_id, - method='POST', - endpoint='/tracker/{}/{}/{}/{}/{}/{}/model/run_parameters/get'.format( - self.job_id, - self.component_name, - self.task_id, - self.task_version, - self.role, - self.party_id), - json_body=json_body) - if response['retcode'] != RetCode.SUCCESS: - raise Exception(f"create table meta failed:{response['retmsg']}") - else: - return response["data"] - - def log_output_data_info(self, data_name: str, table_namespace: str, table_name: str): - LOGGER.info("Request save job {} task {} {} on {} {} data {} info".format(self.job_id, - self.task_id, - self.task_version, - self.role, - self.party_id, - data_name)) - request_body = dict() - request_body["data_name"] = data_name - request_body["table_namespace"] = table_namespace - request_body["table_name"] = table_name - response = api_utils.local_api(job_id=self.job_id, - method='POST', - endpoint='/tracker/{}/{}/{}/{}/{}/{}/output_data_info/save'.format( - self.job_id, - self.component_name, - self.task_id, - self.task_version, - self.role, - self.party_id), - json_body=request_body) - if response['retcode'] != RetCode.SUCCESS: - raise Exception(f"log output data info error, response code: {response['retcode']}, msg: {response['retmsg']}") - - def get_output_data_info(self, data_name=None): - LOGGER.info("Request read job {} task {} {} on {} {} data {} info".format(self.job_id, - self.task_id, - self.task_version, - self.role, - self.party_id, - data_name)) - request_body = dict() - request_body["data_name"] = data_name - response = api_utils.local_api(job_id=self.job_id, - method='POST', - endpoint='/tracker/{}/{}/{}/{}/{}/{}/output_data_info/read'.format( - self.job_id, - self.component_name, - self.task_id, - self.task_version, - self.role, - self.party_id), - json_body=request_body) - if response["retcode"] == RetCode.SUCCESS and "data" in response: - return response["data"] - else: - return None - - def log_component_summary(self, summary_data: dict): - LOGGER.info("Request save job {} task {} {} on {} {} component summary".format(self.job_id, - self.task_id, - self.task_version, - self.role, - self.party_id)) - request_body = dict() - request_body["summary"] = summary_data - response = api_utils.local_api(job_id=self.job_id, - method='POST', - endpoint='/tracker/{}/{}/{}/{}/{}/{}/summary/save'.format( - self.job_id, - self.component_name, - self.task_id, - self.task_version, - self.role, - self.party_id), - json_body=request_body) - if response['retcode'] != RetCode.SUCCESS: - raise Exception(f"log component summary error, response code: {response['retcode']}, msg: {response['retmsg']}") diff --git a/python/fate_flow/scheduling_apps/initiator_app.py b/python/fate_flow/scheduling_apps/initiator_app.py deleted file mode 100644 index 4217272dc..000000000 --- a/python/fate_flow/scheduling_apps/initiator_app.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from flask import request - -from fate_flow.db.db_models import Task -from fate_flow.operation.job_saver import JobSaver -from fate_flow.scheduler.dag_scheduler import DAGScheduler -from fate_flow.utils.api_utils import get_json_result - - -# apply initiator for control operation - - -@manager.route('////stop/', methods=['POST']) -def stop_job(job_id, role, party_id, stop_status): - retcode, retmsg = DAGScheduler.stop_job(job_id=job_id, role=role, party_id=party_id, stop_status=stop_status) - return get_json_result(retcode=retcode, retmsg=retmsg) - - -@manager.route('////rerun', methods=['POST']) -def rerun_job(job_id, role, party_id): - DAGScheduler.set_job_rerun(job_id=job_id, initiator_role=role, initiator_party_id=party_id, - component_name=request.json.get("component_name"), - force=request.json.get("force", False), - auto=False) - #todo: 判断状态 - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('///////report', methods=['POST']) -def report_task(job_id, component_name, task_id, task_version, role, party_id): - task_info = {} - task_info.update(request.json) - task_info.update({ - "job_id": job_id, - "task_id": task_id, - "task_version": task_version, - "role": role, - "party_id": party_id, - }) - JobSaver.update_task(task_info=task_info, report=True) - if task_info.get("party_status"): - JobSaver.update_status(Task, task_info) - return get_json_result(retcode=0, retmsg='success') diff --git a/python/fate_flow/scheduling_apps/operation_app.py b/python/fate_flow/scheduling_apps/operation_app.py deleted file mode 100644 index 3d787225c..000000000 --- a/python/fate_flow/scheduling_apps/operation_app.py +++ /dev/null @@ -1,55 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from flask import request - -from fate_flow.utils import job_utils -from fate_flow.utils.api_utils import get_json_result, error_response -from fate_arch.common import file_utils - - -@manager.route('/job_config/get', methods=['POST']) -def get_config(): - kwargs = {} - job_configuration = None - - for i in ('job_id', 'role', 'party_id'): - if request.json.get(i) is None: - return error_response(400, f"'{i}' is required.") - kwargs[i] = str(request.json[i]) - - for i in ('component_name', 'task_id', 'task_version'): - if request.json.get(i) is None: - break - kwargs[i] = str(request.json[i]) - else: - try: - job_configuration = job_utils.get_task_using_job_conf(**kwargs) - except Exception: - pass - - if job_configuration is None: - job_configuration = job_utils.get_job_configuration(kwargs['job_id'], kwargs['role'], kwargs['party_id']) - - if job_configuration is None: - return error_response(404, 'Job not found.') - - return get_json_result(data=job_configuration.to_dict()) - - -@manager.route('/json_conf/load', methods=['POST']) -def load_json_conf(): - job_conf = file_utils.load_json_conf(request.json.get("config_path")) - return get_json_result(data=job_conf) diff --git a/python/fate_flow/scheduling_apps/party_app.py b/python/fate_flow/scheduling_apps/party_app.py deleted file mode 100644 index 57582ce14..000000000 --- a/python/fate_flow/scheduling_apps/party_app.py +++ /dev/null @@ -1,239 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from flask import request - -from fate_flow.controller.job_controller import JobController -from fate_flow.controller.task_controller import TaskController -from fate_flow.entity import RetCode -from fate_flow.entity.types import TaskCleanResourceType -from fate_flow.manager.dependence_manager import DependenceManager -from fate_flow.manager.resource_manager import ResourceManager -from fate_flow.operation.job_saver import JobSaver -from fate_flow.utils.api_utils import get_json_result, create_job_request_check -from fate_flow.utils.task_utils import task_request_proxy - - -@manager.route('////create', methods=['POST']) -@create_job_request_check -def create_job(job_id, role, party_id): - try: - result = JobController.create_job(job_id=job_id, role=role, party_id=int(party_id), job_info=request.json) - return get_json_result(retcode=0, retmsg='success', data=result) - except RuntimeError as e: - return get_json_result(retcode=RetCode.OPERATING_ERROR, retmsg=str(e), data={"job_id": job_id}) - - -@manager.route('////component/inheritance/check', methods=['POST']) -def component_inheritance_check(job_id, role, party_id): - job = JobSaver.query_job(job_id=job_id, role=role, party_id=party_id)[0] - component_list = DependenceManager.component_check(job, check_type="inheritance") - return get_json_result(data=component_list) - - -@manager.route('////component/rerun/check', methods=['POST']) -def component_rerun_check(job_id, role, party_id): - job = JobSaver.query_job(job_id=job_id, role=role, party_id=party_id)[0] - component_list = DependenceManager.component_check(job, check_type="rerun") - return get_json_result(data=component_list) - - -@manager.route('////dependence/check', methods=['POST']) -def check_dependence(job_id, role, party_id): - job = JobSaver.query_job(job_id=job_id, role=role, party_id=party_id)[0] - status = DependenceManager.check_job_dependence(job) - if status: - return get_json_result(retcode=0, retmsg='success') - else: - return get_json_result(retcode=RetCode.RUNNING, - retmsg=f"check for job {job_id} dependence failed, " - f"dependencies are being installed automatically, it may take a few minutes") - - -@manager.route('////resource/apply', methods=['POST']) -def apply_resource(job_id, role, party_id): - status = ResourceManager.apply_for_job_resource(job_id=job_id, role=role, party_id=int(party_id)) - if status: - return get_json_result(retcode=0, retmsg='success') - else: - return get_json_result(retcode=RetCode.OPERATING_ERROR, retmsg=f"apply for job {job_id} resource failed") - - -@manager.route('////resource/return', methods=['POST']) -def return_resource(job_id, role, party_id): - status = ResourceManager.return_job_resource(job_id=job_id, role=role, party_id=int(party_id)) - if status: - return get_json_result(retcode=0, retmsg='success') - else: - return get_json_result(retcode=RetCode.OPERATING_ERROR, retmsg=f"apply for job {job_id} resource failed") - - -@manager.route('////start', methods=['POST']) -def start_job(job_id, role, party_id): - JobController.start_job(job_id=job_id, role=role, party_id=int(party_id), extra_info=request.json) - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('////align', methods=['POST']) -def align_job_args(job_id, role, party_id): - JobController.align_job_args(job_info=request.json, role=role, party_id=party_id, job_id=job_id) - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('////update', methods=['POST']) -def update_job(job_id, role, party_id): - job_info = {} - job_info.update(request.json) - job_info.update({ - "job_id": job_id, - "role": role, - "party_id": party_id - }) - JobController.update_job(job_info=job_info) - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('////parameter/update', methods=['POST']) -def update_parameters(job_id, role, party_id): - JobController.update_parameter(job_id=job_id, role=role, party_id=party_id, updated_parameters=request.json) - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('////status/', methods=['POST']) -def job_status(job_id, role, party_id, status): - job_info = request.json - # some value of job_info is initiator, should be updated - job_info.update({ - "job_id": job_id, - "role": role, - "party_id": party_id, - "status": status - }) - if JobController.update_job_status(job_info=job_info): - return get_json_result(retcode=0, retmsg='success') - else: - return get_json_result(retcode=RetCode.NOT_EFFECTIVE, retmsg="update job status does not take effect") - - -@manager.route('////model', methods=['POST']) -def save_pipelined_model(job_id, role, party_id): - JobController.save_pipelined_model(job_id=job_id, role=role, party_id=party_id) - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('////stop/', methods=['POST']) -def stop_job(job_id, role, party_id, stop_status): - kill_status, kill_details = JobController.stop_jobs(job_id=job_id, stop_status=stop_status, role=role, party_id=party_id) - return get_json_result(retcode=RetCode.SUCCESS if kill_status else RetCode.EXCEPTION_ERROR, - retmsg='success' if kill_status else 'failed', - data=kill_details) - - -@manager.route('////clean', methods=['POST']) -def clean(job_id, role, party_id): - JobController.clean_job(job_id=job_id, role=role, party_id=party_id, roles=request.json) - return get_json_result(retcode=0, retmsg='success') - - -# Control API for task -@manager.route('///////create', methods=['POST']) -def create_task(job_id, component_name, task_id, task_version, role, party_id): - JobController.initialize_task(role, party_id, request.json) - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('///////start', methods=['POST']) -@task_request_proxy(filter_local=True) -def start_task(job_id, component_name, task_id, task_version, role, party_id): - TaskController.start_task(job_id, component_name, task_id, task_version, role, party_id, **request.json) - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('///////report', methods=['POST']) -def report_task(job_id, component_name, task_id, task_version, role, party_id): - task_info = {} - task_info.update(request.json) - task_info.update({ - "job_id": job_id, - "task_id": task_id, - "task_version": task_version, - "role": role, - "party_id": party_id, - }) - TaskController.update_task(task_info=task_info) - if task_info.get("party_status"): - if not TaskController.update_task_status(task_info=task_info): - return get_json_result(retcode=RetCode.NOT_EFFECTIVE, retmsg="update job status does not take effect") - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('///////update', methods=['POST']) -def update_task(job_id, component_name, task_id, task_version, role, party_id): - task_info = {} - task_info.update(request.json) - task_info.update({ - "job_id": job_id, - "task_id": task_id, - "task_version": task_version, - "role": role, - "party_id": party_id, - }) - TaskController.update_task(task_info=task_info) - return get_json_result(retcode=0, retmsg='success') - - -@manager.route('///////collect', methods=['POST']) -def collect_task(job_id, component_name, task_id, task_version, role, party_id): - task_info = TaskController.collect_task(job_id=job_id, component_name=component_name, task_id=task_id, task_version=task_version, role=role, party_id=party_id) - if task_info: - return get_json_result(retcode=RetCode.SUCCESS, retmsg="success", data=task_info) - else: - return get_json_result(retcode=RetCode.OPERATING_ERROR, retmsg="query task failed") - - -@manager.route('///////status/', methods=['POST']) -def task_status(job_id, component_name, task_id, task_version, role, party_id, status): - task_info = {} - task_info.update({ - "job_id": job_id, - "task_id": task_id, - "task_version": task_version, - "role": role, - "party_id": party_id, - "status": status - }) - if TaskController.update_task_status(task_info=task_info): - return get_json_result(retcode=0, retmsg='success') - else: - return get_json_result(retcode=RetCode.NOT_EFFECTIVE, retmsg="update job status does not take effect") - - -@manager.route('///////stop/', methods=['POST']) -@task_request_proxy() -def stop_task(job_id, component_name, task_id, task_version, role, party_id, stop_status): - tasks = JobSaver.query_task(job_id=job_id, task_id=task_id, task_version=task_version, role=role, party_id=int(party_id)) - kill_status = True - for task in tasks: - kill_status = kill_status & TaskController.stop_task(task=task, stop_status=stop_status, is_asynchronous=request.json.get("is_asynchronous")) - return get_json_result(retcode=RetCode.SUCCESS if kill_status else RetCode.EXCEPTION_ERROR, - retmsg='success' if kill_status else 'failed') - - -@manager.route('///////clean/', methods=['POST']) -def clean_task(job_id, component_name, task_id, task_version, role, party_id, content_type): - TaskController.clean_task(job_id=job_id, task_id=task_id, task_version=task_version, role=role, party_id=int(party_id), content_type=TaskCleanResourceType(content_type)) - return get_json_result(retcode=0, retmsg='success') - - diff --git a/python/fate_flow/scheduling_apps/tracker_app.py b/python/fate_flow/scheduling_apps/tracker_app.py deleted file mode 100644 index 774a28bb3..000000000 --- a/python/fate_flow/scheduling_apps/tracker_app.py +++ /dev/null @@ -1,181 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from flask import request - -from fate_arch.common.base_utils import deserialize_b64 - -from fate_flow.model.sync_model import SyncComponent -from fate_flow.operation.job_tracker import Tracker -from fate_flow.pipelined_model.pipelined_model import PipelinedModel -from fate_flow.pipelined_model.pipelined_component import PipelinedComponent -from fate_flow.settings import ENABLE_MODEL_STORE -from fate_flow.utils.api_utils import get_json_result, validate_request -from fate_flow.utils.model_utils import gen_party_model_id - - -@manager.route('///////metric_data/save', - methods=['POST']) -def save_metric_data(job_id, component_name, task_version, task_id, role, party_id): - request_data = request.json - tracker = Tracker(job_id=job_id, component_name=component_name, task_id=task_id, task_version=task_version, - role=role, party_id=party_id) - metrics = [deserialize_b64(metric) for metric in request_data['metrics']] - tracker.save_metric_data(metric_namespace=request_data['metric_namespace'], metric_name=request_data['metric_name'], - metrics=metrics, job_level=request_data['job_level']) - return get_json_result() - - -@manager.route('///////metric_meta/save', - methods=['POST']) -def save_metric_meta(job_id, component_name, task_version, task_id, role, party_id): - request_data = request.json - tracker = Tracker(job_id=job_id, component_name=component_name, task_id=task_id, task_version=task_version, - role=role, party_id=party_id) - metric_meta = deserialize_b64(request_data['metric_meta']) - tracker.save_metric_meta(metric_namespace=request_data['metric_namespace'], metric_name=request_data['metric_name'], - metric_meta=metric_meta, job_level=request_data['job_level']) - return get_json_result() - - -@manager.route('///////table_meta/create', - methods=['POST']) -def create_table_meta(job_id, component_name, task_version, task_id, role, party_id): - request_data = request.json - tracker = Tracker(job_id=job_id, component_name=component_name, task_id=task_id, task_version=task_version, - role=role, party_id=party_id) - tracker.save_table_meta(request_data) - return get_json_result() - - -@manager.route('///////table_meta/get', - methods=['POST']) -def get_table_meta(job_id, component_name, task_version, task_id, role, party_id): - request_data = request.json - tracker = Tracker(job_id=job_id, component_name=component_name, task_id=task_id, task_version=task_version, - role=role, party_id=party_id) - table_meta_dict = tracker.get_table_meta(request_data) - return get_json_result(data=table_meta_dict) - - -@manager.route('///////model/save', - methods=['POST']) -@manager.route('///////component_model/save', - methods=['POST']) -@validate_request('model_id', 'model_version', 'component_model') -def save_component_model(job_id, component_name, task_version, task_id, role, party_id): - party_model_id = gen_party_model_id(request.json['model_id'], role, party_id) - model_version = request.json['model_version'] - - pipelined_model = PipelinedModel(party_model_id, model_version) - pipelined_model.write_component_model(request.json['component_model']) - - if ENABLE_MODEL_STORE: - sync_component = SyncComponent( - party_model_id=party_model_id, - model_version=model_version, - component_name=component_name, - ) - # no need to test sync_component.remote_exists() - sync_component.upload() - - return get_json_result() - - -@manager.route('///////model/get', - methods=['POST']) -@manager.route('///////component_model/get', - methods=['POST']) -@validate_request('model_id', 'model_version', 'search_model_alias') -def get_component_model(job_id, component_name, task_version, task_id, role, party_id): - party_model_id = gen_party_model_id(request.json['model_id'], role, party_id) - model_version = request.json['model_version'] - - if ENABLE_MODEL_STORE: - sync_component = SyncComponent( - party_model_id=party_model_id, - model_version=model_version, - component_name=component_name, - ) - if not sync_component.local_exists() and sync_component.remote_exists(): - sync_component.download() - - pipelined_model = PipelinedModel(party_model_id, model_version) - data = pipelined_model.read_component_model(component_name, request.json['search_model_alias'], False) - - return get_json_result(data=data) - - -@manager.route('///////model/run_parameters/get', - methods=['POST']) -@validate_request('model_id', 'model_version') -def get_component_model_run_parameters(job_id, component_name, task_version, task_id, role, party_id): - pipelined_component = PipelinedComponent( - role=role, - party_id=party_id, - model_id=request.json['model_id'], - model_version=request.json['model_version'], - ) - data = pipelined_component.get_run_parameters() - - return get_json_result(data=data) - - -@manager.route('///////output_data_info/save', - methods=['POST']) -def save_output_data_info(job_id, component_name, task_version, task_id, role, party_id): - request_data = request.json - tracker = Tracker(job_id=job_id, component_name=component_name, task_id=task_id, task_version=task_version, - role=role, party_id=party_id) - tracker.insert_output_data_info_into_db(data_name=request_data["data_name"], - table_namespace=request_data["table_namespace"], - table_name=request_data["table_name"]) - return get_json_result() - - -@manager.route('///////output_data_info/read', - methods=['POST']) -def read_output_data_info(job_id, component_name, task_version, task_id, role, party_id): - request_data = request.json - tracker = Tracker(job_id=job_id, component_name=component_name, task_id=task_id, task_version=task_version, - role=role, party_id=party_id) - output_data_infos = tracker.read_output_data_info_from_db(data_name=request_data["data_name"]) - response_data = [] - for output_data_info in output_data_infos: - response_data.append(output_data_info.to_human_model_dict()) - return get_json_result(data=response_data) - - -@manager.route('///////summary/save', - methods=['POST']) -def save_component_summary(job_id: str, component_name: str, task_version: int, task_id: str, role: str, party_id: int): - request_data = request.json - tracker = Tracker(job_id=job_id, component_name=component_name, task_id=task_id, task_version=task_version, - role=role, party_id=party_id) - summary_data = request_data['summary'] - tracker.insert_summary_into_db(summary_data) - return get_json_result() - - -@manager.route('/////output/table', methods=['POST']) -def component_output_data_table(job_id, component_name, role, party_id): - output_data_infos = Tracker.query_output_data_infos(job_id=job_id, component_name=component_name, role=role, party_id=party_id) - if output_data_infos: - return get_json_result(retcode=0, retmsg='success', data=[{'table_name': output_data_info.f_table_name, - 'table_namespace': output_data_info.f_table_namespace, - "data_name": output_data_info.f_data_name - } for output_data_info in output_data_infos]) - else: - return get_json_result(retcode=100, retmsg='No found table, please check if the parameters are correct') diff --git a/python/fate_flow/settings.py b/python/fate_flow/settings.py index 658d9d00d..100932377 100644 --- a/python/fate_flow/settings.py +++ b/python/fate_flow/settings.py @@ -13,155 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import os +# GRPC +GRPC_SERVER_MAX_WORKERS = None # default: (os.cpu_count() or 1) * 5 -from grpc._cython import cygrpc +# Request +HTTP_REQUEST_TIMEOUT = 10 # s +REMOTE_REQUEST_TIMEOUT = 30000 # ms -from fate_arch.computing import ComputingEngine -from fate_arch.common import engine_utils -from fate_arch.common.conf_utils import get_base_config, decrypt_database_config -from fate_flow.utils.base_utils import get_fate_flow_directory -from fate_flow.utils.log_utils import LoggerFactory, getLogger +LOG_DIR = "" +DATA_DIR = "" +MODEL_DIR = "" +JOB_DIR = "" +DEFAULT_FATE_DIR = "" +# sqlite +SQLITE_FILE_DIR = "" +SQLITE_FILE_NAME = "fate_flow_sqlite.db" -# Server -API_VERSION = "v1" -FATE_FLOW_SERVICE_NAME = "fateflow" -SERVER_MODULE = "fate_flow_server.py" -CASBIN_TABLE_NAME = "fate_casbin" -TEMP_DIRECTORY = os.path.join(get_fate_flow_directory(), "temp") -FATE_FLOW_CONF_PATH = os.path.join(get_fate_flow_directory(), "conf") -FATE_FLOW_JOB_DEFAULT_CONFIG_PATH = os.path.join(FATE_FLOW_CONF_PATH, "job_default_config.yaml") -FATE_FLOW_DEFAULT_COMPONENT_REGISTRY_PATH = os.path.join(FATE_FLOW_CONF_PATH, "component_registry.json") -TEMPLATE_INFO_PATH = os.path.join(FATE_FLOW_CONF_PATH, "template_info.yaml") -FATE_VERSION_DEPENDENCIES_PATH = os.path.join(get_fate_flow_directory(), "version_dependencies") -CASBIN_MODEL_CONF = os.path.join(FATE_FLOW_CONF_PATH, "casbin_model.conf") -INCOMPATIBLE_VERSION_CONF = os.path.join(FATE_FLOW_CONF_PATH, "incompatible_version.yaml") -SUBPROCESS_STD_LOG_NAME = "std.log" - -GRPC_SERVER_MAX_WORKERS = None -GRPC_OPTIONS = [ - (cygrpc.ChannelArgKey.max_send_message_length, -1), - (cygrpc.ChannelArgKey.max_receive_message_length, -1), -] - -ERROR_REPORT = True -ERROR_REPORT_WITH_PATH = False - -MAX_TIMESTAMP_INTERVAL = 60 -SESSION_VALID_PERIOD = 7 * 24 * 60 * 60 * 1000 - -REQUEST_TRY_TIMES = 3 -REQUEST_WAIT_SEC = 2 -REQUEST_MAX_WAIT_SEC = 300 - -USE_REGISTRY = get_base_config("use_registry") - -# distribution -DEPENDENT_DISTRIBUTION = get_base_config("dependent_distribution", False) -FATE_FLOW_UPDATE_CHECK = False - -HOST = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("host", "127.0.0.1") -HTTP_PORT = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("http_port") -GRPC_PORT = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("grpc_port") - -NGINX_HOST = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("nginx", {}).get("host") or HOST -NGINX_HTTP_PORT = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("nginx", {}).get("http_port") or HTTP_PORT -NGINX_GRPC_PORT = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("nginx", {}).get("grpc_port") or GRPC_PORT - -RANDOM_INSTANCE_ID = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("random_instance_id", False) - -PROXY = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("proxy") -PROXY_PROTOCOL = get_base_config(FATE_FLOW_SERVICE_NAME, {}).get("protocol") - -ENGINES = engine_utils.get_engines() -IS_STANDALONE = engine_utils.is_standalone() - -DATABASE = decrypt_database_config() -ZOOKEEPER = get_base_config("zookeeper", {}) - -# Registry -ZOOKEEPER_REGISTRY = { - # server - 'flow-server': "/FATE-COMPONENTS/fate-flow", - # model service - 'fateflow': "/FATE-SERVICES/flow/online/transfer/providers", - 'servings': "/FATE-SERVICES/serving/online/publishLoad/providers", -} - -# Engine -IGNORE_RESOURCE_COMPUTING_ENGINE = { - ComputingEngine.LINKIS_SPARK -} - -IGNORE_RESOURCE_ROLES = {"arbiter"} - -SUPPORT_IGNORE_RESOURCE_ENGINES = { - ComputingEngine.EGGROLL, ComputingEngine.STANDALONE -} - -# linkis spark config -LINKIS_EXECUTE_ENTRANCE = "/api/rest_j/v1/entrance/execute" -LINKIS_KILL_ENTRANCE = "/api/rest_j/v1/entrance/execID/kill" -LINKIS_QUERT_STATUS = "/api/rest_j/v1/entrance/execID/status" -LINKIS_SUBMIT_PARAMS = { - "configuration": { - "startup": { - "spark.python.version": "/data/anaconda3/bin/python", - "archives": "hdfs:///apps-data/fate/python.zip#python,hdfs:///apps-data/fate/fate_guest.zip#fate_guest", - "spark.executorEnv.PYTHONPATH": "./fate_guest/python:$PYTHONPATH", - "wds.linkis.rm.yarnqueue": "dws", - "spark.pyspark.python": "python/bin/python" - } - } -} -LINKIS_RUNTYPE = "py" -LINKIS_LABELS = {"tenant": "fate"} - -# Endpoint -FATE_FLOW_MODEL_TRANSFER_ENDPOINT = "/v1/model/transfer" -FATE_MANAGER_GET_NODE_INFO_ENDPOINT = "/fate-manager/api/site/secretinfo" -FATE_MANAGER_NODE_CHECK_ENDPOINT = "/fate-manager/api/site/checksite" -FATE_BOARD_DASHBOARD_ENDPOINT = "/index.html#/dashboard?job_id={}&role={}&party_id={}" - -# Logger -LoggerFactory.set_directory(os.path.join(get_fate_flow_directory(), "logs", "fate_flow")) -# {CRITICAL: 50, FATAL:50, ERROR:40, WARNING:30, WARN:30, INFO:20, DEBUG:10, NOTSET:0} -LoggerFactory.LEVEL = 10 - -stat_logger = getLogger("fate_flow_stat") -detect_logger = getLogger("fate_flow_detect") -access_logger = getLogger("fate_flow_access") -database_logger = getLogger("fate_flow_database") - -# Switch -# upload -UPLOAD_DATA_FROM_CLIENT = True - -# authentication -AUTHENTICATION_CONF = get_base_config("authentication", {}) - -PARTY_ID = get_base_config("party_id", "") - -# client -CLIENT_AUTHENTICATION = AUTHENTICATION_CONF.get("client", {}).get("switch", False) -HTTP_APP_KEY = AUTHENTICATION_CONF.get("client", {}).get("http_app_key") -HTTP_SECRET_KEY = AUTHENTICATION_CONF.get("client", {}).get("http_secret_key") - -# site -SITE_AUTHENTICATION = AUTHENTICATION_CONF.get("site", {}).get("switch", False) - -# permission -PERMISSION_CONF = get_base_config("permission", {}) -PERMISSION_SWITCH = PERMISSION_CONF.get("switch") -COMPONENT_PERMISSION = PERMISSION_CONF.get("component") -DATASET_PERMISSION = PERMISSION_CONF.get("dataset") - -HOOK_MODULE = get_base_config("hook_module") -HOOK_SERVER_NAME = get_base_config("hook_server_name") - -ENABLE_MODEL_STORE = get_base_config('enable_model_store', False) - -REMOTE_LOAD_CONF = True -EXTRA_MODEL_DIR = get_fate_flow_directory('model') +# Client Manager +APP_TOKEN_LENGTH = 16 +ADMIN_ID = "admin" +ADMIN_KEY = "fate_flow_admin" diff --git a/python/fate_flow/tests/api_tests/data_access_test.py b/python/fate_flow/tests/api_tests/data_access_test.py deleted file mode 100644 index 6211300b0..000000000 --- a/python/fate_flow/tests/api_tests/data_access_test.py +++ /dev/null @@ -1,63 +0,0 @@ -import os -import time -import unittest - -import requests - -from fate_flow.entity.run_status import JobStatus -from fate_flow.utils.base_utils import get_fate_flow_directory -from fate_flow.settings import API_VERSION, HOST, HTTP_PORT - - -class TestDataAccess(unittest.TestCase): - def setUp(self): - self.data_dir = os.path.join(get_fate_flow_directory(), "examples", "data") - self.upload_guest_config = {"file": os.path.join(self.data_dir, "breast_hetero_guest.csv"), "head": 1, - "partition": 10, "namespace": "experiment", - "table_name": "breast_hetero_guest", "use_local_data": 0, 'drop': 1, 'backend': 0, "id_delimiter": ',', } - self.upload_host_config = {"file": os.path.join(self.data_dir, "breast_hetero_host.csv"), "head": 1, - "partition": 10, "namespace": "experiment", - "table_name": "breast_hetero_host", "use_local_data": 0, 'drop': 1, 'backend': 0, "id_delimiter": ',', } - self.download_config = {"output_path": os.path.join(get_fate_flow_directory(), - "fate_flow/fate_flow_unittest_breast_b.csv"), - "namespace": "experiment", - "table_name": "breast_hetero_guest"} - self.server_url = "http://{}:{}/{}".format(HOST, HTTP_PORT, API_VERSION) - - def test_upload_guest(self): - response = requests.post("/".join([self.server_url, 'data', 'upload']), json=self.upload_guest_config) - self.assertTrue(response.status_code in [200, 201]) - self.assertTrue(int(response.json()['retcode']) == 0) - job_id = response.json()['jobId'] - for i in range(60): - response = requests.post("/".join([self.server_url, 'job', 'query']), json={'job_id': job_id}) - self.assertTrue(int(response.json()['retcode']) == 0) - if response.json()['data'][0]['f_status'] == JobStatus.SUCCESS: - break - time.sleep(1) - - def test_upload_host(self): - response = requests.post("/".join([self.server_url, 'data', 'upload']), json=self.upload_host_config) - self.assertTrue(response.status_code in [200, 201]) - self.assertTrue(int(response.json()['retcode']) == 0) - job_id = response.json()['jobId'] - for i in range(60): - response = requests.post("/".join([self.server_url, 'job', 'query']), json={'job_id': job_id}) - self.assertTrue(int(response.json()['retcode']) == 0) - if response.json()['data'][0]['f_status'] == JobStatus.SUCCESS: - break - time.sleep(1) - - def test_upload_history(self): - response = requests.post("/".join([self.server_url, 'data', 'upload/history']), json={'limit': 2}) - self.assertTrue(response.status_code in [200, 201]) - self.assertTrue(int(response.json()['retcode']) == 0) - - def test_download(self): - response = requests.post("/".join([self.server_url, 'data', 'download']), json=self.download_config) - self.assertTrue(response.status_code in [200, 201]) - self.assertTrue(int(response.json()['retcode']) == 0) - - -if __name__ == '__main__': - unittest.main() diff --git a/python/fate_flow/tests/api_tests/set_unittest_config.py b/python/fate_flow/tests/api_tests/set_unittest_config.py deleted file mode 100644 index 2d958f6a6..000000000 --- a/python/fate_flow/tests/api_tests/set_unittest_config.py +++ /dev/null @@ -1,22 +0,0 @@ -import argparse -import json -import os - -def set_config(guest_party_id, host_party_id): - os.makedirs('./jobs', exist_ok=True) - with open(os.path.join('./jobs', 'party_info.json'), 'w') as fw: - json.dump({ - 'guest': guest_party_id, - 'host': host_party_id - }, fw) - - -if __name__ == '__main__': - arg_parser = argparse.ArgumentParser() - arg_parser.add_argument("guest_party_id", type=int, help="please input guest party id") - arg_parser.add_argument("host_party_id", type=int, help="please input host party id") - args = arg_parser.parse_args() - guest_party_id = args.guest_party_id - host_party_id = args.host_party_id - set_config(guest_party_id, host_party_id) - diff --git a/python/fate_flow/tests/api_tests/table_test.py b/python/fate_flow/tests/api_tests/table_test.py deleted file mode 100644 index e148ced89..000000000 --- a/python/fate_flow/tests/api_tests/table_test.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -import time -import unittest - -import requests - -from fate_flow.entity.run_status import JobStatus -from fate_flow.utils.base_utils import get_fate_flow_directory -from fate_flow.settings import API_VERSION, HOST, HTTP_PORT - - -server_url = "http://{}:{}/{}".format(HOST, HTTP_PORT, API_VERSION) - - -class TestTable(unittest.TestCase): - def setUp(self): - self.data_dir = os.path.join(get_fate_flow_directory(), "examples", "data") - self.upload_config = {"file": os.path.join(self.data_dir, "breast_hetero_guest.csv"), "head": 1, - "partition": 10, "namespace": "fate_flow_test_table_breast_hetero", - "table_name": "breast_hetero_guest", "use_local_data": 0, 'drop': 1, 'backend': 0, "id_delimiter": ','} - - - def test_upload_guest(self): - response = requests.post("/".join([server_url, 'data', 'upload']), json=self.upload_config) - self.assertTrue(response.status_code in [200, 201]) - self.assertTrue(int(response.json()['retcode']) == 0) - job_id = response.json()['jobId'] - for i in range(60): - response = requests.post("/".join([server_url, 'job', 'query']), json={'job_id': job_id}) - self.assertTrue(int(response.json()['retcode']) == 0) - if response.json()['data'][0]['f_status'] == JobStatus.SUCCESS: - break - time.sleep(1) - self.assertTrue(response.json()['data'][0]['f_status'] == JobStatus.SUCCESS) - - response = test_table_info() - self.assertTrue(response.status_code in [200, 201]) - self.assertTrue(int(response.json()['retcode']) == 0) - - response = test_table_delete() - self.assertTrue(response.status_code in [200, 201]) - self.assertTrue(int(response.json()['retcode']) == 0) - - - -def test_table_info(): - response = requests.post("/".join([server_url, 'table', 'table_info']), - json={'table_name': 'breast_hetero_guest', - 'namespace': 'fate_flow_test_table_breast_hetero'}) - return response - - -def test_table_delete(): - # submit - response = requests.post("/".join([server_url, 'table', 'delete']), - json={'table_name': 'breast_hetero_guest', - 'namespace': 'fate_flow_test_table_breast_hetero'}) - return response - - -if __name__ == '__main__': - unittest.main() - - diff --git a/python/fate_flow/tests/api_tests/tracking_test.py b/python/fate_flow/tests/api_tests/tracking_test.py deleted file mode 100644 index 5e8351e5a..000000000 --- a/python/fate_flow/tests/api_tests/tracking_test.py +++ /dev/null @@ -1,96 +0,0 @@ -import json -import os -import time -import unittest - -import requests - -from fate_flow.entity.run_status import EndStatus, JobStatus -from fate_arch.common.file_utils import load_json_conf -from fate_flow.utils.base_utils import get_fate_flow_python_directory -from fate_flow.settings import API_VERSION, HOST, HTTP_PORT,IS_STANDALONE - -WORK_MODE = 1 if not IS_STANDALONE else 0 - -class TestTracking(unittest.TestCase): - def setUp(self): - self.sleep_time = 10 - self.success_job_dir = './jobs/' - self.dsl_path = 'fate_flow/examples/test_hetero_lr_job_dsl.json' - self.config_path = 'fate_flow/examples/test_hetero_lr_job_conf.json' - self.test_component_name = 'hetero_feature_selection_0' - self.server_url = "http://{}:{}/{}".format(HOST, HTTP_PORT, API_VERSION) - self.party_info = load_json_conf(os.path.abspath(os.path.join('./jobs', 'party_info.json'))) if WORK_MODE else None - self.guest_party_id = self.party_info['guest'] if WORK_MODE else 9999 - self.host_party_id = self.party_info['host'] if WORK_MODE else 10000 - - def test_tracking(self): - with open(os.path.join(get_fate_flow_python_directory(), self.dsl_path), 'r') as f: - dsl_data = json.load(f) - with open(os.path.join(get_fate_flow_python_directory(), self.config_path), 'r') as f: - config_data = json.load(f) - config_data[ "initiator"]["party_id"] = self.guest_party_id - config_data["role"] = { - "guest": [self.guest_party_id], - "host": [self.host_party_id], - "arbiter": [self.host_party_id] - } - response = requests.post("/".join([self.server_url, 'job', 'submit']), - json={'job_dsl': dsl_data, 'job_runtime_conf': config_data}) - self.assertTrue(response.status_code in [200, 201]) - self.assertTrue(int(response.json()['retcode']) == 0) - job_id = response.json()['jobId'] - job_info = {'f_status': 'running'} - for i in range(60): - response = requests.post("/".join([self.server_url, 'job', 'query']), json={'job_id': job_id, 'role': 'guest'}) - self.assertTrue(response.status_code in [200, 201]) - job_info = response.json()['data'][0] - if EndStatus.contains(job_info['f_status']): - break - time.sleep(self.sleep_time) - print('waiting job run success, the job has been running for {}s'.format((i+1)*self.sleep_time)) - self.assertTrue(job_info['f_status'] == JobStatus.SUCCESS) - os.makedirs(self.success_job_dir, exist_ok=True) - with open(os.path.join(self.success_job_dir, job_id), 'w') as fw: - json.dump(job_info, fw) - self.assertTrue(os.path.exists(os.path.join(self.success_job_dir, job_id))) - - # test_component_parameters - test_component(self, 'component/parameters') - - # test_component_metric_all - test_component(self, 'component/metric/all') - - # test_component_metric - test_component(self, 'component/metrics') - - # test_component_output_model - test_component(self, 'component/output/model') - - # test_component_output_data_download - test_component(self, 'component/output/data') - - # test_component_output_data_download - test_component(self, 'component/output/data/download') - - # test_job_data_view - test_component(self, 'job/data_view') - - -def test_component(self, fun): - job_id = os.listdir(os.path.abspath(os.path.join(self.success_job_dir)))[-1] - job_info = load_json_conf(os.path.abspath(os.path.join(self.success_job_dir, job_id))) - data = {'job_id': job_id, 'role': job_info['f_role'], 'party_id': job_info['f_party_id'], 'component_name': self.test_component_name} - if 'download' in fun: - response = requests.get("/".join([self.server_url, "tracking", fun]), json=data, stream=True) - self.assertTrue(response.status_code in [200, 201]) - else: - response = requests.post("/".join([self.server_url, 'tracking', fun]), json=data) - self.assertTrue(response.status_code in [200, 201]) - self.assertTrue(int(response.json()['retcode']) == 0) - - -if __name__ == '__main__': - unittest.main() - - diff --git a/python/fate_flow/tests/check_all_api.py b/python/fate_flow/tests/check_all_api.py deleted file mode 100644 index 62801326e..000000000 --- a/python/fate_flow/tests/check_all_api.py +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import sys -import requests -import json -from fate_flow.settings import HOST, API_VERSION, HTTP_PORT - -fate_flow_server_host = 'http://{}:{}/{}'.format(HOST, HTTP_PORT, API_VERSION) -job_id = sys.argv[1] -role = sys.argv[2] -party_id = int(sys.argv[3]) -base_request_data = {'job_id': job_id, 'role': role, 'party_id': party_id} -print('job id is {}'.format(job_id)) -# data view -print('job data view') -response = requests.post('{}/tracking/job/data_view'.format(fate_flow_server_host), json=base_request_data) -print(response.json()) -response = requests.post('{}/job/data/view/query'.format(fate_flow_server_host), json=base_request_data) -print(response.json()) - -# dependency -print('dependency') -response = requests.post('{}/pipeline/dag/dependency'.format(fate_flow_server_host), - json={'job_id': job_id, 'role': role, 'party_id': party_id}) -dependency_response = response.json() -print(json.dumps(dependency_response)) -print() -for component_name in dependency_response['data']['component_list']: - print('component name is {}'.format(component_name)) - base_request_data['component_name'] = component_name - # metrics - print('metrics') - response = requests.post('{}/tracking/component/metrics'.format(fate_flow_server_host), json=base_request_data) - print(response.json()) - print('metrics return {}'.format(response.json())) - print() - if response.json()['retcode'] == 0 and response.json()['retmsg'] != "no data": - for metric_namespace, metric_names in response.json()['data'].items(): - base_request_data['metric_namespace'] = metric_namespace - for metric_name in metric_names: - base_request_data['metric_name'] = metric_name - response = requests.post('{}/tracking/component/metric_data'.format(fate_flow_server_host), json=base_request_data) - if response.json()['retcode'] == 0 : - print('{} {} metric data:'.format(metric_namespace, metric_name)) - print(response.json()) - else: - print('{} {} no metric data!'.format(metric_namespace, metric_name)) - print() - - # parameters - print('parameters') - response = requests.post('{}/tracking/component/parameters'.format(fate_flow_server_host), json=base_request_data) - print(response.json()) - print('parameters retcode {}'.format(response.json()['retcode'])) - # model - print('output model') - response = requests.post('{}/tracking/component/output/model'.format(fate_flow_server_host), json=base_request_data) - print(response.json()) - print('output model retcode {}'.format(response.json()['retcode'])) - # data - print('output data') - response = requests.post('{}/tracking/component/output/data'.format(fate_flow_server_host), json=base_request_data) - print(response.json()) - print('output data retcode {}'.format(response.json()['retcode'])) - print() - response = requests.post('{}/tracking/component/output/data/table'.format(fate_flow_server_host), json=base_request_data) - print(response.json()) diff --git a/python/fate_flow/tests/grpc/client.py b/python/fate_flow/tests/grpc/client.py deleted file mode 100644 index 8d2e7dc54..000000000 --- a/python/fate_flow/tests/grpc/client.py +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import json -import sys -import time - -import grpc - -from fate_arch.protobuf.python import proxy_pb2_grpc -from fate_flow.utils.log_utils import audit_logger, schedule_logger -from fate_flow.utils.grpc_utils import wrap_grpc_packet, gen_routing_metadata - - -def get_command_federation_channel(host, port): - print(f"connect {host}:{port}") - channel = grpc.insecure_channel('{}:{}'.format(host, port)) - stub = proxy_pb2_grpc.DataTransferServiceStub(channel) - return channel, stub - - -def remote_api(host, port, job_id, method, endpoint, src_party_id, dest_party_id, src_role, json_body, api_version="v1", - overall_timeout=30*1000, try_times=3): - endpoint = f"/{api_version}{endpoint}" - json_body['src_role'] = src_role - json_body['src_party_id'] = src_party_id - _packet = wrap_grpc_packet(json_body, method, endpoint, src_party_id, dest_party_id, job_id, - overall_timeout=overall_timeout) - print(_packet) - _routing_metadata = gen_routing_metadata(src_party_id=src_party_id, dest_party_id=dest_party_id) - exception = None - for t in range(try_times): - try: - channel, stub = get_command_federation_channel(host, port) - _return, _call = stub.unaryCall.with_call(_packet, metadata=_routing_metadata, timeout=(overall_timeout/1000)) - audit_logger(job_id).info("grpc api response: {}".format(_return)) - channel.close() - response = json.loads(_return.body.value) - return response - except Exception as e: - exception = e - schedule_logger(job_id).warning(f"remote request {endpoint} error, sleep and try again") - time.sleep(2 * (t+1)) - else: - tips = 'Please check rollSite and fateflow network connectivity' - raise Exception('{}rpc request error: {}'.format(tips, exception)) - -host = sys.argv[1] -port = int(sys.argv[2]) -src_role = sys.argv[3] -src_party_id = sys.argv[4] -dest_party_id = sys.argv[5] -response = remote_api(host, port, "test_job_command", "POST", "/version/get", src_party_id, dest_party_id, src_role, {"src_role": src_role, "src_party_id": src_party_id}) -print(response) - diff --git a/python/fate_flow/tests/grpc/server.py b/python/fate_flow/tests/grpc/server.py deleted file mode 100644 index eb27e7001..000000000 --- a/python/fate_flow/tests/grpc/server.py +++ /dev/null @@ -1,95 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import time -import sys - -import grpc -import requests -from grpc._cython import cygrpc - -from fate_arch.protobuf.python import basic_meta_pb2, proxy_pb2, proxy_pb2_grpc -from fate_arch.common.base_utils import json_dumps, json_loads - -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.db.job_default_config import JobDefaultConfig -from fate_flow.settings import FATE_FLOW_SERVICE_NAME, stat_logger, HOST, GRPC_PORT - -from fate_flow.tests.grpc.xthread import ThreadPoolExecutor - - -def wrap_grpc_packet(json_body, http_method, url, src_party_id, dst_party_id, job_id=None, overall_timeout=None): - overall_timeout = JobDefaultConfig.remote_request_timeout if overall_timeout is None else overall_timeout - _src_end_point = basic_meta_pb2.Endpoint(ip=HOST, port=GRPC_PORT) - _src = proxy_pb2.Topic(name=job_id, partyId="{}".format(src_party_id), role=FATE_FLOW_SERVICE_NAME, callback=_src_end_point) - _dst = proxy_pb2.Topic(name=job_id, partyId="{}".format(dst_party_id), role=FATE_FLOW_SERVICE_NAME, callback=None) - _task = proxy_pb2.Task(taskId=job_id) - _command = proxy_pb2.Command(name=FATE_FLOW_SERVICE_NAME) - _conf = proxy_pb2.Conf(overallTimeout=overall_timeout) - _meta = proxy_pb2.Metadata(src=_src, dst=_dst, task=_task, command=_command, operator=http_method, conf=_conf) - _data = proxy_pb2.Data(key=url, value=bytes(json_dumps(json_body), 'utf-8')) - return proxy_pb2.Packet(header=_meta, body=_data) - - -def get_url(_suffix): - return "http://{}:{}/{}".format(RuntimeConfig.JOB_SERVER_HOST, RuntimeConfig.HTTP_PORT, _suffix.lstrip('/')) - - -class UnaryService(proxy_pb2_grpc.DataTransferServiceServicer): - @staticmethod - def unaryCall(_request, context): - packet = _request - header = packet.header - _suffix = packet.body.key - param_bytes = packet.body.value - param = bytes.decode(param_bytes) - job_id = header.task.taskId - src = header.src - dst = header.dst - method = header.operator - param_dict = json_loads(param) - param_dict['src_party_id'] = str(src.partyId) - source_routing_header = [] - for key, value in context.invocation_metadata(): - source_routing_header.append((key, value)) - stat_logger.info(f"grpc request routing header: {source_routing_header}") - - action = getattr(requests, method.lower(), None) - if action: - print(_suffix) - else: - pass - resp_json = {"status": "test"} - import time - print("sleep") - time.sleep(60) - return wrap_grpc_packet(resp_json, method, _suffix, dst.partyId, src.partyId, job_id) - -thread_pool_executor = ThreadPoolExecutor(max_workers=5) -print(f"start grpc server pool on {thread_pool_executor._max_workers} max workers") -server = grpc.server(thread_pool_executor, - options=[(cygrpc.ChannelArgKey.max_send_message_length, -1), - (cygrpc.ChannelArgKey.max_receive_message_length, -1)]) - -proxy_pb2_grpc.add_DataTransferServiceServicer_to_server(UnaryService(), server) -server.add_insecure_port("{}:{}".format("127.0.0.1", 9360)) -server.start() - -try: - while True: - time.sleep(60 * 60 * 24) -except KeyboardInterrupt: - server.stop(0) - sys.exit(0) diff --git a/python/fate_flow/tests/grpc/xthread.py b/python/fate_flow/tests/grpc/xthread.py deleted file mode 100644 index 0c215af75..000000000 --- a/python/fate_flow/tests/grpc/xthread.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2009 Brian Quinlan. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Implements ThreadPoolExecutor.""" - -__author__ = 'Brian Quinlan (brian@sweetapp.com)' - -import atexit -from concurrent.futures import _base -import itertools -import queue -import threading -import weakref -import os - -# Workers are created as daemon threads. This is done to allow the interpreter -# to exit when there are still idle threads in a ThreadPoolExecutor's thread -# pool (i.e. shutdown() was not called). However, allowing workers to die with -# the interpreter has two undesirable properties: -# - The workers would still be running during interpreter shutdown, -# meaning that they would fail in unpredictable ways. -# - The workers could be killed while evaluating a work item, which could -# be bad if the callable being evaluated has external side-effects e.g. -# writing to a file. -# -# To work around this problem, an exit handler is installed which tells the -# workers to exit when their work queues are empty and then waits until the -# threads finish. - -_threads_queues = weakref.WeakKeyDictionary() -_shutdown = False - -def _python_exit(): - global _shutdown - _shutdown = True - items = list(_threads_queues.items()) - for t, q in items: - q.put(None) - for t, q in items: - t.join() - -atexit.register(_python_exit) - -class _WorkItem(object): - def __init__(self, future, fn, args, kwargs): - self.future = future - self.fn = fn - self.args = args - self.kwargs = kwargs - - def run(self): - if not self.future.set_running_or_notify_cancel(): - return - - try: - result = self.fn(*self.args, **self.kwargs) - except BaseException as exc: - self.future.set_exception(exc) - # Break a reference cycle with the exception 'exc' - self = None - else: - self.future.set_result(result) - -def _worker(executor_reference, work_queue): - try: - while True: - print(f"work queue size {work_queue.qsize()}") - work_item = work_queue.get(block=True) - # print("get work item") - if work_item is not None: - work_item.run() - # Delete references to object. See issue16284 - del work_item - continue - executor = executor_reference() - # Exit if: - # - The interpreter is shutting down OR - # - The executor that owns the worker has been collected OR - # - The executor that owns the worker has been shutdown. - if _shutdown or executor is None or executor._shutdown: - # Notice other workers - work_queue.put(None) - return - del executor - except BaseException: - _base.LOGGER.critical('Exception in worker', exc_info=True) - -class ThreadPoolExecutor(_base.Executor): - - # Used to assign unique thread names when thread_name_prefix is not supplied. - _counter = itertools.count().__next__ - - def __init__(self, max_workers=None, thread_name_prefix=''): - """Initializes a new ThreadPoolExecutor instance. - - Args: - max_workers: The maximum number of threads that can be used to - execute the given calls. - thread_name_prefix: An optional name prefix to give our threads. - """ - if max_workers is None: - # Use this number because ThreadPoolExecutor is often - # used to overlap I/O instead of CPU work. - max_workers = (os.cpu_count() or 1) * 5 - if max_workers <= 0: - raise ValueError("max_workers must be greater than 0") - - self._max_workers = max_workers - self._work_queue = queue.Queue() - self._threads = set() - self._shutdown = False - self._shutdown_lock = threading.Lock() - self._thread_name_prefix = (thread_name_prefix or - ("ThreadPoolExecutor-%d" % self._counter())) - - def submit(self, fn, *args, **kwargs): - with self._shutdown_lock: - if self._shutdown: - raise RuntimeError('cannot schedule new futures after shutdown') - - f = _base.Future() - w = _WorkItem(f, fn, args, kwargs) - - self._work_queue.put(w) - self._adjust_thread_count() - return f - submit.__doc__ = _base.Executor.submit.__doc__ - - def _adjust_thread_count(self): - # When the executor gets lost, the weakref callback will wake up - # the worker threads. - def weakref_cb(_, q=self._work_queue): - q.put(None) - # TODO(bquinlan): Should avoid creating new threads if there are more - # idle threads than items in the work queue. - num_threads = len(self._threads) - if num_threads < self._max_workers: - thread_name = '%s_%d' % (self._thread_name_prefix or self, - num_threads) - t = threading.Thread(name=thread_name, target=_worker, - args=(weakref.ref(self, weakref_cb), - self._work_queue)) - t.daemon = True - t.start() - self._threads.add(t) - _threads_queues[t] = self._work_queue - else: - print(f"the number of max workers {self._max_workers} has been exceeded, worker queue size {self._work_queue.qsize()}") - - def shutdown(self, wait=True): - with self._shutdown_lock: - self._shutdown = True - self._work_queue.put(None) - if wait: - for t in self._threads: - t.join() - shutdown.__doc__ = _base.Executor.shutdown.__doc__ \ No newline at end of file diff --git a/python/fate_flow/tests/inference_request.py b/python/fate_flow/tests/inference_request.py deleted file mode 100644 index 581fd632c..000000000 --- a/python/fate_flow/tests/inference_request.py +++ /dev/null @@ -1,54 +0,0 @@ -import grpc -import time -import json -import sys -import uuid - -from fate_arch.protobuf.python import inference_service_pb2 -from fate_arch.protobuf.python import inference_service_pb2_grpc -import threading - - -def run(address): - ths = [] - with grpc.insecure_channel(address) as channel: - for i in range(1): - th = threading.Thread(target=send, args=(channel,)) - ths.append(th) - st = int(time.time()) - for th in ths: - th.start() - for th in ths: - th.join() - et = int(time.time()) - - -def process_response(call_future): - print(call_future.result()) - - -def send(channel): - stub = inference_service_pb2_grpc.InferenceServiceStub(channel) - request = inference_service_pb2.InferenceMessage() - request_data = dict() - request_data['serviceId'] = 'xxxxxxxxx' - request_data['applyId'] = '' - # request_data['modelId'] = 'arbiter-10000#guest-10000#host-10000#model' # You can specify the model id this way - # request_data['modelVersion'] = 'acd3e1807a1211e9969aacde48001122' # You can specify the model version this way - request_data['caseid'] = uuid.uuid1().hex - - feature_data = dict() - feature_data['fid1'] = 5.1 - feature_data['fid2'] = 6.2 - feature_data['fid3'] = 7.6 - request_data['featureData'] = feature_data - request_data['sendToRemoteFeatureData'] = feature_data - - print(json.dumps(request_data, indent=4)) - - request.body = json.dumps(request_data).encode(encoding='utf-8') - print(stub.inference(request)) - - -if __name__ == '__main__': - run(sys.argv[1]) diff --git a/python/fate_flow/tests/misc/DataIOMeta.pb b/python/fate_flow/tests/misc/DataIOMeta.pb deleted file mode 100644 index a1bf72c18..000000000 Binary files a/python/fate_flow/tests/misc/DataIOMeta.pb and /dev/null differ diff --git a/python/fate_flow/tests/misc/define_meta.yaml b/python/fate_flow/tests/misc/define_meta.yaml deleted file mode 100644 index 47e4d5abe..000000000 --- a/python/fate_flow/tests/misc/define_meta.yaml +++ /dev/null @@ -1,56 +0,0 @@ -describe: This is the model definition meta -component_define: - dataio_0: - module_name: DataIO - statistic_0: - module_name: DataStatistics - hetero_feature_binning_0: - module_name: HeteroFeatureBinning - hetero_feature_selection_0: - module_name: HeteroFeatureSelection - feature_scale_0: - module_name: FeatureScale - hetero_lr_0: - module_name: HeteroLR - pipeline: - module_name: Pipeline -model_proto: - dataio_0: - dataio: - DataIOMeta: DataIOMeta - DataIOParam: DataIOParam - default: - DataIOMeta: DataIOMeta - DataIOParam: DataIOParam - statistic_0: - model: - StatisticMeta: StatisticMeta - StatisticParam: ModelParam - hetero_feature_binning_0: - binning_model: - FeatureBinningMeta: FeatureBinningMeta - FeatureBinningParam: FeatureBinningParam - hetero_feature_selection_0: - selected: - FeatureSelectionMeta: FeatureSelectionMeta - FeatureSelectionParam: FeatureSelectionParam - default: - FeatureSelectionMeta: FeatureSelectionMeta - FeatureSelectionParam: FeatureSelectionParam - feature_scale_0: - feature_scale: - ScaleMeta: ScaleMeta - ScaleParam: ScaleParam - default: - ScaleMeta: ScaleMeta - ScaleParam: ScaleParam - hetero_lr_0: - hetero_lr: - HeteroLogisticRegressionMeta: LRModelMeta - HeteroLogisticRegressionParam: LRModelParam - default: - HeteroLogisticRegressionMeta: LRModelMeta - HeteroLogisticRegressionParam: LRModelParam - pipeline: - pipeline: - Pipeline: Pipeline diff --git a/python/fate_flow/tests/model_tests/checkpoint_test.py b/python/fate_flow/tests/model_tests/checkpoint_test.py deleted file mode 100644 index 70ddc2ba8..000000000 --- a/python/fate_flow/tests/model_tests/checkpoint_test.py +++ /dev/null @@ -1,197 +0,0 @@ -import unittest -from unittest.mock import patch - -import hashlib -from pathlib import Path -from datetime import datetime -from collections import deque -from tempfile import TemporaryDirectory - -from ruamel import yaml - -from fate_flow.model import checkpoint - - -model_string = (Path(__file__).parent.parent / 'misc' / 'DataIOMeta.pb').read_bytes() -sha1 = hashlib.sha1(model_string).hexdigest() -buffer_name = 'DataIOMeta' -model_buffers = { - 'my_model': checkpoint.parse_proto_object(buffer_name, model_string), -} -data = yaml.dump({ - 'step_index': 123, - 'step_name': 'foobar', - 'create_time': '2021-07-08T07:51:01.963423', - 'models': { - 'my_model': { - 'filename': 'my_model.pb', - 'sha1': sha1, - 'buffer_name': buffer_name, - }, - }, -}, Dumper=yaml.RoundTripDumper) - - -class TestCheckpoint(unittest.TestCase): - - def setUp(self): - self.tmpdir = TemporaryDirectory() - self.checkpoint = checkpoint.Checkpoint(Path(self.tmpdir.name), 123, 'foobar') - self.filepath = self.checkpoint.directory / 'my_model.pb' - - def tearDown(self): - self.tmpdir.cleanup() - - def test_path(self): - directory = Path(self.tmpdir.name) / '123#foobar' - self.assertEqual(self.checkpoint.directory, directory) - self.assertEqual(self.checkpoint.database, directory / 'database.yaml') - - def test_save_checkpoint(self): - self.assertTrue(self.checkpoint.directory.exists()) - self.assertFalse(self.checkpoint.available) - self.assertFalse(self.filepath.exists()) - self.assertIsNone(self.checkpoint.create_time) - - self.checkpoint.save(model_buffers) - self.assertTrue(self.checkpoint.available) - self.assertTrue(self.filepath.exists()) - self.assertIsNotNone(self.checkpoint.create_time) - - self.assertEqual(self.checkpoint.database.read_text('utf8'), - data.replace('2021-07-08T07:51:01.963423', self.checkpoint.create_time.isoformat()), 1) - self.assertEqual(self.filepath.read_bytes(), model_string) - - def test_read_checkpoint(self): - self.assertTrue(self.checkpoint.directory.exists()) - self.assertFalse(self.checkpoint.available) - self.assertFalse(self.filepath.exists()) - - self.filepath.write_bytes(model_string) - self.assertFalse(self.checkpoint.available) - - self.checkpoint.database.write_text(data, 'utf8') - self.assertTrue(self.checkpoint.available) - self.assertIsNone(self.checkpoint.create_time) - - self.assertEqual(self.checkpoint.read(), model_buffers) - self.assertEqual(self.checkpoint.step_index, 123) - self.assertEqual(self.checkpoint.step_name, 'foobar') - self.assertEqual(self.checkpoint.create_time, datetime.fromisoformat('2021-07-08T07:51:01.963423')) - - def test_remove_checkpoint(self): - self.checkpoint.save(model_buffers) - self.checkpoint.database.write_text(data, 'utf8') - self.checkpoint.remove() - - self.assertTrue(self.checkpoint.directory.exists()) - self.assertFalse(self.filepath.exists()) - self.assertFalse(self.checkpoint.available) - self.assertIsNone(self.checkpoint.create_time) - - def test_read_checkpoint_step_index_or_step_name_not_match(self): - self.filepath.write_bytes(model_string) - self.checkpoint.database.write_text(data.replace('123', '233', 1), 'utf8') - with self.assertRaisesRegex(ValueError, 'Checkpoint may be incorrect: step_index or step_name dose not match.'): - self.checkpoint.read() - - def test_read_checkpoint_no_pb_file(self): - self.checkpoint.database.write_text(data, 'utf8') - with self.assertRaisesRegex(FileNotFoundError, 'Checkpoint is incorrect: protobuf file not found.'): - self.checkpoint.read() - - def test_read_checkpoint_hash_not_match(self): - self.filepath.write_bytes(model_string) - self.checkpoint.database.write_text(data.replace(sha1, 'abcdef', 1), 'utf8') - with self.assertRaisesRegex(ValueError, 'Checkpoint may be incorrect: hash dose not match.'): - self.checkpoint.read() - - -class TestCheckpointManager(unittest.TestCase): - - def setUp(self): - self.tmpdir = TemporaryDirectory() - with patch('fate_flow.model.checkpoint.get_project_base_directory', return_value=self.tmpdir.name): - self.checkpoint_manager = checkpoint.CheckpointManager('job_id', 'role', 1000, 'model_id', 'model_version') - - def tearDown(self): - self.tmpdir.cleanup() - - def test_directory(self): - self.assertEqual(self.checkpoint_manager.directory, - Path(self.tmpdir.name) / 'model_local_cache' / - 'role#1000#model_id' / 'model_version' / 'checkpoint' / 'pipeline') - - def test_load_checkpoints_from_disk(self): - for x in range(1, 51): - directory = self.checkpoint_manager.directory / f'{x}#foobar{x}' - directory.mkdir(0o755) - - (directory / 'my_model.pb').write_bytes(model_string) - (directory / 'database.yaml').write_text( - data.replace('123', str(x), 1).replace('foobar', f'foobar{x}', 1), 'utf8') - - self.checkpoint_manager.load_checkpoints_from_disk() - self.assertEqual(self.checkpoint_manager.checkpoints_number, 50) - self.assertEqual(self.checkpoint_manager.latest_step_index, 50) - self.assertEqual(self.checkpoint_manager.latest_step_name, 'foobar50') - self.assertEqual(self.checkpoint_manager.latest_checkpoint.read(), model_buffers) - - def test_checkpoint_index(self): - for x in range(1, 101, 2): - directory = self.checkpoint_manager.directory / f'{x}#foobar{x}' - directory.mkdir(0o755) - - (directory / 'my_model.pb').write_bytes(model_string) - (directory / 'database.yaml').write_text( - data.replace('123', str(x), 1).replace('foobar', f'foobar{x}', 1), 'utf8') - - self.checkpoint_manager.load_checkpoints_from_disk() - self.assertEqual(list(self.checkpoint_manager.number_indexed_checkpoints.keys()), - list(range(1, 101, 2))) - self.assertEqual(list(self.checkpoint_manager.name_indexed_checkpoints.keys()), - [f'foobar{x}' for x in range(1, 101, 2)]) - - for x in range(1, 101, 2): - _checkpoint = self.checkpoint_manager.get_checkpoint_by_index(x) - self.assertIs(self.checkpoint_manager.get_checkpoint_by_name(f'foobar{x}'), _checkpoint) - self.assertEqual(_checkpoint.step_index, x) - self.assertEqual(_checkpoint.step_name, f'foobar{x}') - self.assertIsNone(_checkpoint.create_time) - - _model_buffers = _checkpoint.read() - self.assertEqual(_checkpoint.step_index, x) - self.assertEqual(_checkpoint.step_name, f'foobar{x}') - self.assertEqual(_checkpoint.create_time.isoformat(), '2021-07-08T07:51:01.963423') - - def test_new_checkpoint(self): - self.checkpoint_manager.checkpoints = deque(maxlen=10) - - for x in range(1, 31): - _checkpoint = self.checkpoint_manager.new_checkpoint(x, f'foobar{x}') - _checkpoint.save(model_buffers) - self.assertEqual(self.checkpoint_manager.latest_step_index, x) - self.assertEqual(self.checkpoint_manager.latest_step_name, f'foobar{x}') - - self.assertEqual(self.checkpoint_manager.checkpoints_number, 10) - self.assertEqual(len(list(self.checkpoint_manager.directory.rglob('my_model.pb'))), 10) - self.assertEqual(len(list(self.checkpoint_manager.directory.rglob('database.yaml'))), 10) - self.assertEqual(len(list(self.checkpoint_manager.directory.rglob('.lock'))), 10) - self.assertEqual(len(list(self.checkpoint_manager.directory.glob('*'))), 30) - - def test_clean(self): - for x in range(10): - _checkpoint = self.checkpoint_manager.new_checkpoint(x, f'foobar{x}') - _checkpoint.save(model_buffers) - - self.assertEqual(self.checkpoint_manager.checkpoints_number, 10) - self.assertEqual(len(list(self.checkpoint_manager.directory.glob('*'))), 10) - - self.checkpoint_manager.clean() - self.assertEqual(self.checkpoint_manager.checkpoints_number, 0) - self.assertTrue(self.checkpoint_manager.directory.exists()) - self.assertEqual(len(list(self.checkpoint_manager.directory.glob('*'))), 0) - - -if __name__ == '__main__': - unittest.main() diff --git a/python/fate_flow/tests/model_tests/pipelined_model_test.py b/python/fate_flow/tests/model_tests/pipelined_model_test.py deleted file mode 100644 index 9bace86b0..000000000 --- a/python/fate_flow/tests/model_tests/pipelined_model_test.py +++ /dev/null @@ -1,187 +0,0 @@ -import unittest -from unittest.mock import patch - -import os -import io -import shutil -import hashlib -import concurrent.futures -from pathlib import Path -from copy import deepcopy -from zipfile import ZipFile - -from ruamel import yaml - -from fate_flow.pipelined_model.pipelined_model import PipelinedModel -from fate_flow.settings import TEMP_DIRECTORY - - -with open(Path(__file__).parent.parent / 'misc' / 'define_meta.yaml', encoding='utf8') as _f: - data_define_meta = yaml.safe_load(_f) -args_update_component_meta = [ - 'dataio_0', - 'DataIO', - 'dataio', - { - 'DataIOMeta': 'DataIOMeta', - 'DataIOParam': 'DataIOParam', - }, -] - - -class TestPipelinedModel(unittest.TestCase): - - def setUp(self): - shutil.rmtree(TEMP_DIRECTORY, True) - - self.pipelined_model = PipelinedModel('foobar', 'v1') - shutil.rmtree(self.pipelined_model.model_path, True) - self.pipelined_model.create_pipelined_model() - - with open(self.pipelined_model.define_meta_path, 'w', encoding='utf8') as f: - yaml.dump(data_define_meta, f) - - def tearDown(self): - shutil.rmtree(TEMP_DIRECTORY, True) - shutil.rmtree(self.pipelined_model.model_path, True) - - def test_write_read_file_same_time(self): - fw = open(self.pipelined_model.define_meta_path, 'r+', encoding='utf8') - self.assertEqual(yaml.safe_load(fw), data_define_meta) - fw.seek(0) - fw.write('foobar') - - with open(self.pipelined_model.define_meta_path, encoding='utf8') as fr: - self.assertEqual(yaml.safe_load(fr), data_define_meta) - - fw.truncate() - - with open(self.pipelined_model.define_meta_path, encoding='utf8') as fr: - self.assertEqual(fr.read(), 'foobar') - - fw.seek(0) - fw.write('abc') - fw.close() - - with open(self.pipelined_model.define_meta_path, encoding='utf8') as fr: - self.assertEqual(fr.read(), 'abcbar') - - def test_update_component_meta_with_changes(self): - with patch('ruamel.yaml.dump', side_effect=yaml.dump) as yaml_dump: - self.pipelined_model.update_component_meta( - 'dataio_0', 'DataIO_v0', 'dataio', { - 'DataIOMeta': 'DataIOMeta_v0', - 'DataIOParam': 'DataIOParam_v0', - } - ) - yaml_dump.assert_called_once() - - with open(self.pipelined_model.define_meta_path, encoding='utf8') as tmp: - define_index = yaml.safe_load(tmp) - - _data = deepcopy(data_define_meta) - _data['component_define']['dataio_0']['module_name'] = 'DataIO_v0' - _data['model_proto']['dataio_0']['dataio'] = { - 'DataIOMeta': 'DataIOMeta_v0', - 'DataIOParam': 'DataIOParam_v0', - } - - self.assertEqual(define_index, _data) - - def test_update_component_meta_without_changes(self): - with open(self.pipelined_model.define_meta_path, 'w', encoding='utf8') as f: - yaml.dump(data_define_meta, f, Dumper=yaml.RoundTripDumper) - - with patch('ruamel.yaml.dump', side_effect=yaml.dump) as yaml_dump: - self.pipelined_model.update_component_meta(*args_update_component_meta) - yaml_dump.assert_not_called() - - with open(self.pipelined_model.define_meta_path, encoding='utf8') as tmp: - define_index = yaml.safe_load(tmp) - self.assertEqual(define_index, data_define_meta) - - def test_update_component_meta_multi_thread(self): - with patch('ruamel.yaml.safe_load', side_effect=yaml.safe_load) as yaml_load, \ - patch('ruamel.yaml.dump', side_effect=yaml.dump) as yaml_dump, \ - concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor: - for _ in range(100): - executor.submit(self.pipelined_model.update_component_meta, *args_update_component_meta) - self.assertEqual(yaml_load.call_count, 100) - self.assertEqual(yaml_dump.call_count, 0) - - with open(self.pipelined_model.define_meta_path, encoding='utf8') as tmp: - define_index = yaml.safe_load(tmp) - self.assertEqual(define_index, data_define_meta) - - def test_update_component_meta_empty_file(self): - open(self.pipelined_model.define_meta_path, 'w').close() - with self.assertRaisesRegex(ValueError, 'Invalid meta file'): - self.pipelined_model.update_component_meta(*args_update_component_meta) - - def test_packaging_model(self): - archive_file_path = self.pipelined_model.packaging_model() - self.assertEqual(archive_file_path, self.pipelined_model.archive_model_file_path) - self.assertTrue(Path(archive_file_path).is_file()) - self.assertTrue(Path(archive_file_path + '.sha1').is_file()) - - with ZipFile(archive_file_path) as z: - with io.TextIOWrapper(z.open('define/define_meta.yaml'), encoding='utf8') as f: - define_index = yaml.safe_load(f) - self.assertEqual(define_index, data_define_meta) - - with open(archive_file_path, 'rb') as f, open(archive_file_path + '.sha1', encoding='utf8') as g: - sha1 = hashlib.sha1(f.read()).hexdigest() - sha1_orig = g.read().strip() - self.assertEqual(sha1, sha1_orig) - - def test_packaging_model_not_exists(self): - shutil.rmtree(self.pipelined_model.model_path, True) - with self.assertRaisesRegex(FileNotFoundError, 'Can not found foobar v1 model local cache'): - self.pipelined_model.packaging_model() - - def test_unpack_model(self): - archive_file_path = self.pipelined_model.packaging_model() - self.assertTrue(Path(archive_file_path + '.sha1').is_file()) - - shutil.rmtree(self.pipelined_model.model_path, True) - self.assertFalse(Path(self.pipelined_model.model_path).exists()) - - self.pipelined_model.unpack_model(archive_file_path) - with open(self.pipelined_model.define_meta_path, encoding='utf8') as tmp: - define_index = yaml.safe_load(tmp) - self.assertEqual(define_index, data_define_meta) - - def test_unpack_model_local_cache_exists(self): - archive_file_path = self.pipelined_model.packaging_model() - - with self.assertRaisesRegex(FileExistsError, 'Model foobar v1 local cache already existed'): - self.pipelined_model.unpack_model(archive_file_path) - - def test_unpack_model_no_hash_file(self): - archive_file_path = self.pipelined_model.packaging_model() - Path(archive_file_path + '.sha1').unlink() - self.assertFalse(Path(archive_file_path + '.sha1').exists()) - - shutil.rmtree(self.pipelined_model.model_path, True) - self.assertFalse(os.path.exists(self.pipelined_model.model_path)) - - self.pipelined_model.unpack_model(archive_file_path) - with open(self.pipelined_model.define_meta_path, encoding='utf8') as tmp: - define_index = yaml.safe_load(tmp) - self.assertEqual(define_index, data_define_meta) - - def test_unpack_model_hash_not_match(self): - archive_file_path = self.pipelined_model.packaging_model() - self.assertTrue(Path(archive_file_path + '.sha1').is_file()) - with open(archive_file_path + '.sha1', 'w', encoding='utf8') as f: - f.write('abc123') - - shutil.rmtree(self.pipelined_model.model_path, True) - self.assertFalse(Path(self.pipelined_model.model_path).exists()) - - with self.assertRaisesRegex(ValueError, 'Hash not match.'): - self.pipelined_model.unpack_model(archive_file_path) - - -if __name__ == '__main__': - unittest.main() diff --git a/python/fate_flow/tests/model_tests/services_test.py b/python/fate_flow/tests/model_tests/services_test.py deleted file mode 100644 index 16009b3c3..000000000 --- a/python/fate_flow/tests/model_tests/services_test.py +++ /dev/null @@ -1,151 +0,0 @@ -import os -import time -import unittest -from unittest.mock import patch - -from kazoo.client import KazooClient -from kazoo.exceptions import NodeExistsError, NoNodeError - -from fate_flow.db import db_services -from fate_flow.errors.error_services import * -from fate_flow.db.db_models import DB, MachineLearningModelInfo as MLModel -from fate_flow import settings - - -model_download_url = 'http://127.0.0.1:9380/v1/model/transfer/arbiter-10000_guest-9999_host-10000_model/202105060929263278441' -escaped_model_download_url = '/FATE-SERVICES/flow/online/transfer/providers/http%3A%2F%2F127.0.0.1%3A9380%2Fv1%2Fmodel%2Ftransfer%2Farbiter-10000_guest-9999_host-10000_model%2F202105060929263278441' - - -class TestZooKeeperDB(unittest.TestCase): - - def setUp(self): - # required environment: ZOOKEEPER_HOSTS - # optional environment: ZOOKEEPER_USERNAME, ZOOKEEPER_PASSWORD - config = { - 'hosts': os.environ['ZOOKEEPER_HOSTS'].split(','), - 'use_acl': False, - } - username = os.environ.get('ZOOKEEPER_USERNAME') - password = os.environ.get('ZOOKEEPER_PASSWORD') - if username and password: - config.update({ - 'use_acl': True, - 'username': username, - 'password': password, - }) - - with patch.object(db_services.ServiceRegistry, 'USE_REGISTRY', 'ZooKeeper'), \ - patch.object(db_services.ServiceRegistry, 'ZOOKEEPER', config): - self.service_db = db_services.service_db() - - def test_services_db(self): - self.assertEqual(type(self.service_db), db_services.ZooKeeperDB) - self.assertNotEqual(type(self.service_db), db_services.FallbackDB) - self.assertEqual(type(self.service_db.client), KazooClient) - - def test_zookeeper_not_configured(self): - with patch.object(db_services.ServiceRegistry, 'USE_REGISTRY', True), \ - patch.object(db_services.ServiceRegistry, 'ZOOKEEPER', {'hosts': None}), \ - self.assertRaisesRegex(ZooKeeperNotConfigured, ZooKeeperNotConfigured.message): - db_services.service_db() - - def test_missing_zookeeper_username_or_password(self): - with patch.object(db_services.ServiceRegistry, 'USE_REGISTRY', True), \ - patch.object(db_services.ServiceRegistry, 'ZOOKEEPER', { - 'hosts': ['127.0.0.1:2281'], - 'use_acl': True, - }), self.assertRaisesRegex( - MissingZooKeeperUsernameOrPassword, MissingZooKeeperUsernameOrPassword.message): - db_services.service_db() - - def test_get_znode_path(self): - self.assertEqual(self.service_db._get_znode_path('fateflow', model_download_url), escaped_model_download_url) - - def test_crud(self): - self.service_db._insert('fateflow', model_download_url) - self.assertIn(model_download_url, self.service_db.get_urls('fateflow')) - - self.service_db._delete('fateflow', model_download_url) - self.assertNotIn(model_download_url, self.service_db.get_urls('fateflow')) - - def test_insert_exists_node(self): - self.service_db._delete('servings', 'http://foo/bar') - self.service_db._insert('servings', 'http://foo/bar') - - with self.assertRaises(NodeExistsError): - self.service_db.client.create(self.service_db._get_znode_path('servings', 'http://foo/bar'), makepath=True) - - self.service_db._insert('servings', 'http://foo/bar') - self.service_db._delete('servings', 'http://foo/bar') - - def test_delete_not_exists_node(self): - self.service_db._delete('servings', 'http://foo/bar') - - with self.assertRaises(NoNodeError): - self.service_db.client.delete(self.service_db._get_znode_path('servings', 'http://foo/bar')) - - self.service_db._delete('servings', 'http://foo/bar') - - def test_connection_closed(self): - self.service_db._insert('fateflow', model_download_url) - self.assertIn(model_download_url, self.service_db.get_urls('fateflow')) - - self.service_db.client.stop() - self.service_db.client.start() - self.assertNotIn(model_download_url, self.service_db.get_urls('fateflow')) - - def test_register_models(self): - try: - os.remove(DB.database) - except FileNotFoundError: - pass - - MLModel.create_table() - for x in range(1, 101): - job_id = str(time.time()) - model = MLModel( - f_role='host', f_party_id='100', f_job_id=job_id, - f_model_id=f'foobar#{x}', f_model_version=job_id, - f_initiator_role='host', f_work_mode=0 - ) - model.save(force_insert=True) - self.assertEqual(db_services.models_group_by_party_model_id_and_model_version().count(), 100) - - with patch.object(self.service_db, '_insert') as insert: - self.service_db.register_models() - self.assertEqual(insert.call_count, 100) - with patch.object(self.service_db, '_delete') as delete: - self.service_db.unregister_models() - self.assertEqual(delete.call_count, 100) - - os.remove(DB.database) - - -class TestFallbackDB(unittest.TestCase): - - def setUp(self): - with patch.object(db_services.ServiceRegistry, 'USE_REGISTRY', False): - self.service_db = db_services.service_db() - - def test_get_urls(self): - self.assertEqual(self.service_db._get_urls('fateflow'), ['http://127.0.0.1:9380/v1/model/transfer']) - self.assertEqual(self.service_db._get_urls('servings'), ['http://127.0.0.1:8000']) - - def test_crud(self): - self.service_db._insert('fateflow', model_download_url) - self.assertNotIn(model_download_url, self.service_db.get_urls('fateflow')) - - self.service_db._delete('fateflow', model_download_url) - self.assertNotIn(model_download_url, self.service_db.get_urls('fateflow')) - - def test_get_model_download_url(self): - self.assertEqual(db_services.get_model_download_url('foo-111#bar-222', '20210616'), - 'http://127.0.0.1:9380/v1/model/transfer/foo-111_bar-222/20210616') - - def test_not_supported_service(self): - with self.assertRaisesRegex(ServiceNotSupported, 'The service foobar is not supported'): - self.service_db.get_urls('foobar') - - -if __name__ == '__main__': - unittest.main() diff --git a/python/fate_flow/tests/python3_7_modules.csv b/python/fate_flow/tests/python3_7_modules.csv deleted file mode 100644 index dba36cb07..000000000 --- a/python/fate_flow/tests/python3_7_modules.csv +++ /dev/null @@ -1,325 +0,0 @@ -__future__ -__main__ -_dummy_thread -_thread -abc -aifc -argparse -array -ast -asynchat -asyncio -asyncore -atexit -audioop -base64 -bdb -binascii -binhex -bisect -builtins -bz2 -cProfile -calendar -cgi -cgitb -chunk -cmath -cmd -code -codecs -codeop -collections -collections.abc -colorsys -compileall -concurrent.futures -configparser -contextlib -contextvars -copy -copyreg -crypt -csv -ctypes -curses -curses.ascii -curses.panel -curses.textpad -dataclasses -datetime -dbm -dbm.dumb -dbm.gnu -dbm.ndbm -decimal -difflib -dis -distutils -distutils.archive_util -distutils.bcppcompiler -distutils.ccompiler -distutils.cmd -distutils.command -distutils.command.bdist -distutils.command.bdist_dumb -distutils.command.bdist_msi -distutils.command.bdist_packager -distutils.command.bdist_rpm -distutils.command.bdist_wininst -distutils.command.build -distutils.command.build_clib -distutils.command.build_ext -distutils.command.build_py -distutils.command.build_scripts -distutils.command.check -distutils.command.clean -distutils.command.config -distutils.command.install -distutils.command.install_data -distutils.command.install_headers -distutils.command.install_lib -distutils.command.install_scripts -distutils.command.register -distutils.command.sdist -distutils.core -distutils.cygwinccompiler -distutils.debug -distutils.dep_util -distutils.dir_util -distutils.dist -distutils.errors -distutils.extension -distutils.fancy_getopt -distutils.file_util -distutils.filelist -distutils.log -distutils.msvccompiler -distutils.spawn -distutils.sysconfig -distutils.text_file -distutils.unixccompiler -distutils.util -distutils.version -doctest -dummy_threading -email -email.charset -email.contentmanager -email.encoders -email.errors -email.generator -email.header -email.headerregistry -email.iterators -email.message -email.mime -email.parser -email.policy -email.utils -encodings.idna -encodings.mbcs -encodings.utf_8_sig -ensurepip -enum -errno -faulthandler -fcntl -filecmp -fileinput -fnmatch -formatter -fractions -ftplib -functools -gc -getopt -getpass -gettext -glob -grp -gzip -hashlib -heapq -hmac -html -html.entities -html.parser -http -http.client -http.cookiejar -http.cookies -http.server -imaplib -imghdr -imp -importlib -importlib.abc -importlib.machinery -importlib.resources -importlib.util -inspect -io -ipaddress -itertools -json -json.tool -keyword -lib2to3 -linecache -locale -logging -logging.config -logging.handlers -lzma -macpath -mailbox -mailcap -marshal -math -mimetypes -mmap -modulefinder -msilib -msvcrt -multiprocessing -multiprocessing.connection -multiprocessing.dummy -multiprocessing.managers -multiprocessing.pool -multiprocessing.sharedctypes -netrc -nis -nntplib -numbers -operator -optparse -os -os.path -ossaudiodev -parser -pathlib -pdb -pickle -pickletools -pipes -pkgutil -platform -plistlib -poplib -posix -pprint -profile -pstats -pty -pwd -py_compile -pyclbr -pydoc -queue -quopri -random -re -readline -reprlib -resource -rlcompleter -runpy -sched -secrets -select -selectors -shelve -shlex -shutil -signal -site -smtpd -smtplib -sndhdr -socket -socketserver -spwd -sqlite3 -ssl -stat -statistics -string -stringprep -struct -subprocess -sunau -symbol -symtable -sys -sysconfig -syslog -tabnanny -tarfile -telnetlib -tempfile -termios -test -test.support -test.support.script_helper -textwrap -threading -time -timeit -tkinter -tkinter.scrolledtext -tkinter.tix -tkinter.ttk -token -tokenize -trace -traceback -tracemalloc -tty -turtle -turtledemo -types -typing -unicodedata -unittest -unittest.mock -urllib -urllib.error -urllib.parse -urllib.request -urllib.response -urllib.robotparser -uu -uuid -venv -warnings -wave -weakref -webbrowser -winreg -winsound -wsgiref -wsgiref.handlers -wsgiref.headers -wsgiref.simple_server -wsgiref.util -wsgiref.validate -xdrlib -xml -xml.dom -xml.dom.minidom -xml.dom.pulldom -xml.etree.ElementTree -xml.parsers.expat -xml.parsers.expat.errors -xml.parsers.expat.model -xml.sax -xml.sax.handler -xml.sax.saxutils -xml.sax.xmlreader -xmlrpc.client -xmlrpc.server -zipapp -zipfile -zipimport -zlib \ No newline at end of file diff --git a/python/fate_flow/tests/upload_file_to_mysql.py b/python/fate_flow/tests/upload_file_to_mysql.py deleted file mode 100644 index 1a2506271..000000000 --- a/python/fate_flow/tests/upload_file_to_mysql.py +++ /dev/null @@ -1,103 +0,0 @@ -import argparse - -import pymysql - -from fate_arch import storage - -database_config = { - 'user': 'root', - 'passwd': 'fate_dev', - 'host': '127.0.0.1', - 'port': 3306 -} - - -class MysqldbHelper(object): - def __init__(self, host='', user='', passwd='', port='', database=''): - self.host = host - self.user = user - self.password = passwd - self.database = database - self.port = port - self.con = None - self.cur = None - try: - print(host, user, passwd, port, database) - self.con = pymysql.connect(host=self.host, user=self.user, passwd=self.password, port=self.port, db=self.database) - self.cur = self.con.cursor() - except: - print("DataBase connect error,please check the db config.") - - def execute(self, sql): - self.cur.execute(sql) - self.cur.fetchall() - - -def create_db(namespace): - conn = pymysql.connect(host=database_config.get('host'), - port=database_config.get('port'), - user=database_config.get('user'), - password=database_config.get('passwd')) - cursor = conn.cursor() - cursor.execute("create database if not exists {}".format(namespace)) - print('create db {} success'.format(namespace)) - cursor.close() - - -def list_to_str(input_list): - return ','.join(list(map(str, input_list))) - - -def write_to_db(conf, table_name, file_name, namespace, partitions, head): - db = MysqldbHelper(**conf) - table_meta = storage.StorageTableMeta(name=table_name, namespace=namespace) - create_table = 'create table {}(id varchar(50) NOT NULL, features LONGTEXT, PRIMARY KEY(id))'.format(table_name) - db.execute(create_table.format(table_name)) - print('create table {}'.format(table_name)) - - with open(file_name, 'r') as f: - if head: - data_head = f.readline() - header_source_item = data_head.split(',') - table_meta.update_metas(schema={'header': ','.join(header_source_item[1:]).strip(), 'sid': header_source_item[0]}) - n = 0 - count = 0 - while True: - data = list() - lines = f.readlines(12400) - - if lines: - sql = 'REPLACE INTO {}(id, features) VALUES'.format(table_name) - for line in lines: - count += 1 - values = line.replace("\n", "").replace("\t", ",").split(",") - data.append((values[0], list_to_str(values[1:]))) - sql += '("{}", "{}"),'.format(values[0], list_to_str(values[1:])) - sql = ','.join(sql.split(',')[:-1]) + ';' - if n == 0: - table_meta.update_metas(part_of_data=data, partitions=partitions) - n +=1 - db.execute(sql) - db.con.commit() - else: - break - table_meta.update_metas(count=count) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('-n', '--namespace', required=True, type=str, help="namespace") - parser.add_argument('-t', '--table_name', required=True, type=str, help="table_name") - parser.add_argument('-f', '--file_name', required=True, type=str, help="file_name") - parser.add_argument('-p', '--partitions', required=True, type=int, help="partitions") - parser.add_argument('-head', '--head', required=True, type=int, help="head") - - args = parser.parse_args() - namespace = args.namespace - table_name = args.table_name - file_name = args.file_name - partitions = args.partitions - head = args.head - create_db(namespace) - database_config['database'] = namespace - write_to_db(database_config, table_name, file_name, namespace, partitions=partitions, head=head) \ No newline at end of file diff --git a/python/fate_flow/tools/deal_rollsite_audit_log.py b/python/fate_flow/tools/deal_rollsite_audit_log.py deleted file mode 100644 index 925b0d5af..000000000 --- a/python/fate_flow/tools/deal_rollsite_audit_log.py +++ /dev/null @@ -1,242 +0,0 @@ -import os -import json -import sys -import re -import requests -import traceback -import datetime -from deal_rollsite_audit_log_settings import LOG_INDEX, ELASTIC_SEARCH_URL, ELASTIC_SEARCH_AUTH, ELASTIC_SEARCH_USER, ELASTIC_SEARCH_PASSWORD, HOST_ROLE_PARTY_ID - -LOG_PATH = "" -SERVER_IP = None -EXCHANGE_TYPE = "general" - - -def run(): - progress = read_progress() - end_pos = progress.get("end_pos", -1) - last_st_ino = progress.get("st_ino", -1) - last_st_mtime = progress.get("st_mtime", -1) - now_st_ino = os.stat(LOG_PATH).st_ino - print(f"last inode num: {last_st_ino}, mtime: {last_st_mtime}") - print(f"{LOG_PATH} inode num is {now_st_ino}") - if last_st_ino != -1 and last_st_ino != now_st_ino: - # create time not match, log path have change - print(f"last inode num is {last_st_ino}, but now is {now_st_ino}, log path have change, search all pending log") - last_deal_log_path, pending_paths = search_pending_logs(os.path.dirname(LOG_PATH), last_st_ino, last_st_mtime) - print(f"find last deal log path: {last_deal_log_path}") - print(f"find pending paths: {pending_paths}") - deal_log(last_deal_log_path, end_pos) - for pending_path in pending_paths: - deal_log(pending_path, -1) - # reset end pos - end_pos = -1 - end_pos = deal_log(LOG_PATH, end_pos) - progress["end_pos"] = end_pos - progress["st_mtime"] = os.stat(LOG_PATH).st_mtime - progress["st_ino"] = os.stat(LOG_PATH).st_ino - save_progress(progress) - - -def deal_log(LOG_PATH, end_pos): - if not LOG_PATH: - return end_pos - audit_logs = [] - with open(LOG_PATH) as fr: - line_count = get_file_line_count(fr) - print(f"{LOG_PATH} end pos: {end_pos}, line count: {line_count}") - if line_count > end_pos + 1: - fr.seek(end_pos + 1) - while True: - line = fr.readline() - if not line: - break - audit_log = deal_line(line) - merge_audit_log(audit_log, audit_logs) - end_pos = fr.tell() - else: - print(f"{LOG_PATH} no change") - if audit_logs: - bulk_save(audit_logs) - return end_pos - - -def merge_audit_log(audit_log, audit_logs): - if audit_log: - #audit_logs.append(json.dumps({"index": {"_index": "fate_rollsite_exchange_audit"}})) - audit_logs.append('{"index":{}}') - audit_logs.append(json.dumps(audit_log)) - - -def search_pending_logs(log_dir, st_ino, st_mtime): - last_deal_log_path = None - pending_paths = [] - - year_dirs = [os.path.join(log_dir, f) for f in os.listdir(log_dir) if os.path.isdir(os.path.join(log_dir, f))] - year_dirs.sort(key=lambda f: os.stat(f).st_mtime, reverse=True) - for year_dir in year_dirs: - print(f"search year dir: {year_dir}") - month_dirs = [os.path.join(year_dir, f) for f in os.listdir(year_dir) if os.path.isdir(os.path.join(year_dir, f))] - month_dirs.sort(key=lambda f: os.stat(f).st_mtime, reverse=True) - year_search = False - for month_dir in month_dirs: - print(f"search month dir: {month_dir}") - day_dirs = [os.path.join(month_dir, f) for f in os.listdir(month_dir) if os.path.isdir(os.path.join(month_dir, f))] - day_dirs.sort(key=lambda f: os.stat(f).st_mtime, reverse=True) - month_search = False - for day_dir in day_dirs: - print(f"search day dir: {day_dir}") - last_deal_log_path, day_pending_paths = get_pending_logs(day_dir, st_ino, st_mtime) - if day_pending_paths: - print(f"get pending path: {day_pending_paths}") - pending_paths.extend(day_pending_paths) - else: - print(f"{day_dir} no pending path, break") - break - else: - # all day dir have pending_paths - month_search = True - if not month_search: - break - else: - # all day dir have pending_paths - year_search = True - if not year_search: - break - return last_deal_log_path, pending_paths - - -def get_pending_logs(day_dir, st_ino, st_mtime): - pending_paths = [] - st_mtime_match_path = None - for f in os.listdir(day_dir): - f_p = os.path.join(day_dir, f) - if os.path.isfile(f_p) and f.startswith("rollsite-audit.log") and os.stat(f_p).st_mtime >= st_mtime: - if os.stat(f_p).st_ino == st_ino: - st_mtime_match_path = f_p - else: - pending_paths.append(f_p) - return st_mtime_match_path, pending_paths - - -def get_file_line_count(fp): - fp.seek(0, 2) - return fp.tell() - - -def progress_file_path(): - return os.path.join(os.path.dirname(os.path.realpath(__file__)), "deal_rollsite_log_progress.json") - - -def read_progress(): - p_p = progress_file_path() - if not os.path.exists(p_p): - return {} - with open(p_p) as fr: - return json.load(fr) - - -def save_progress(progress): - p_p = progress_file_path() - with open(p_p, "w") as fw: - json.dump(progress, fw, indent=4) - - -def deal_line(src): - #a = "[INFO ][36165610][2021-03-19 20:08:05,935][grpc-server-9370-30,pid:32590,tid:89][audit:87] - task={taskId=202103192007180194594}|src={name=202103192007180194594,partyId=9999,role=fateflow,callback={ip=127.0.0.1,port=9360}}|dst={name=202103192007180194594,partyId=10000,role=fateflow}|command={name=/v1/party/202103192007180194594/arbiter/10000/clean}|operator=POST|conf={overallTimeout=30000}" - meta_data = {} - try: - split_items = src.split(" - ") - meta_line = split_items[1].strip() - meta_data["logTime"] = re.findall("\[.*?\]", split_items[0])[2].strip("[").strip("]") - meta_data["logTime"] = (datetime.datetime.strptime(meta_data["logTime"], "%Y-%m-%d %H:%M:%S,%f") - datetime.timedelta(hours=8)).strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] - for meta_item_str in meta_line.split("|"): - meta_item_key = meta_item_str[:meta_item_str.index("=")] - meta_item_value_str = meta_item_str[meta_item_str.index("=") + 1:] - if meta_item_value_str.find("{") == 0: - meta_item_value = str_to_dict(meta_item_value_str[1:-1]) - else: - meta_item_value = meta_item_value_str - meta_data[meta_item_key] = meta_item_value - meta_data["jobId"] = meta_data["task"]["taskId"] - meta_data["jobDate"] = (datetime.datetime.strptime(meta_data["jobId"][:14], "%Y%m%d%H%M%S") - datetime.timedelta(hours=8)).strftime("%Y-%m-%d %H:%M:%S") - meta_data["server"] = SERVER_IP - meta_data["exchangeType"] = EXCHANGE_TYPE - meta_data["src"]["role"] = "host" if meta_data["src"]["partyId"] in HOST_ROLE_PARTY_ID else "guest" - meta_data["dst"]["role"] = "host" if meta_data["dst"]["partyId"] in HOST_ROLE_PARTY_ID else "guest" - except Exception as e: - traceback.print_exc() - return meta_data - - -def str_to_dict(src): - key = "" - value = "" - current = 1 # 1 for key, 2 for value - - d = {} - i = 0 - while True: - c = src[i] - if c == "{": - j = i + 1 - sub_str = "" - while True: - if src[j] == "}": - j = j + 1 - break - else: - sub_str += src[j] - j = j + 1 - sub = str_to_dict(sub_str) - if current == 2: - d[key] = sub - i = j - else: - if c == "=": - current = 2 - elif c == ",": - d[key] = value - key = "" - value = "" - current = 1 - else: - if current == 1: - key += c - elif current == 2: - value += c - i = i + 1 - if i == len(src): - if key and value: - d[key] = value - break - return d - - -def upload(audit_log): - res = requests.post("/".join([ELASTIC_SEARCH_URL, LOG_INDEX, "_doc"]), json=audit_log) - print(res.json()) - - -def bulk_save(audit_logs): - data = "\n".join(audit_logs) + "\n" - if ELASTIC_SEARCH_AUTH: - res = requests.post("/".join([ELASTIC_SEARCH_URL, LOG_INDEX, "_doc", "_bulk"]), - data=data, - headers={'content-type':'application/json', 'charset':'UTF-8'}, - timeout=(30, 300), - auth=(ELASTIC_SEARCH_USER, ELASTIC_SEARCH_PASSWORD)) - else: - res = requests.post("/".join([ELASTIC_SEARCH_URL, LOG_INDEX, "_doc", "_bulk"]), - data=data, - headers={'content-type':'application/json', 'charset':'UTF-8'}, - timeout=(30, 300)) - print(res.text) - print(res.json()) - - -if __name__ == "__main__": - LOG_PATH = sys.argv[1] - SERVER_IP = sys.argv[2] - EXCHANGE_TYPE = sys.argv[3] - run() \ No newline at end of file diff --git a/python/fate_flow/tools/deal_rollsite_audit_log_settings.py b/python/fate_flow/tools/deal_rollsite_audit_log_settings.py deleted file mode 100644 index aa84a674b..000000000 --- a/python/fate_flow/tools/deal_rollsite_audit_log_settings.py +++ /dev/null @@ -1,8 +0,0 @@ -LOG_INDEX = "fate_rollsite_exchange_audit" - -ELASTIC_SEARCH_URL = "" -ELASTIC_SEARCH_AUTH = False -ELASTIC_SEARCH_USER = "" -ELASTIC_SEARCH_PASSWORD = "" - -HOST_ROLE_PARTY_ID = ["10000", "10001", "10002", "10003", "10004", "10005", "10006", "10017", "10018", "10019"] \ No newline at end of file diff --git a/python/fate_flow/tools/deal_rollsite_log_progress.json b/python/fate_flow/tools/deal_rollsite_log_progress.json deleted file mode 100644 index 389b6a944..000000000 --- a/python/fate_flow/tools/deal_rollsite_log_progress.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "end_pos": 112736, - "st_mtime": 1621579189.3877091, - "st_ino": 169158861 -} \ No newline at end of file diff --git a/python/fate_flow/tools/rollsite_exchange_audit_index.json b/python/fate_flow/tools/rollsite_exchange_audit_index.json deleted file mode 100644 index 2860f3ad7..000000000 --- a/python/fate_flow/tools/rollsite_exchange_audit_index.json +++ /dev/null @@ -1,181 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": "3", - "number_of_replicas": "1" - } - }, - "mappings": { - "properties": { - "command": { - "properties": { - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "conf": { - "properties": { - "overallTimeout": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "dst": { - "properties": { - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "partyId": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "role": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "exchangeType": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "jobId": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "jobDate": { - "type": "date", - "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis", - "ignore_malformed": false - }, - "logTime": { - "type": "date", - "format": "yyyy-MM-dd HH:mm:ss,SSS||yyyy-MM-dd||epoch_millis", - "ignore_malformed": false - }, - "operator": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "server": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "src": { - "properties": { - "callback": { - "properties": { - "ip": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "port": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "partyId": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "role": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - }, - "task": { - "properties": { - "taskId": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - } - } -} \ No newline at end of file diff --git a/python/fate_flow/utils/__init__.py b/python/fate_flow/utils/__init__.py index be3a4ddf9..ae946a49c 100644 --- a/python/fate_flow/utils/__init__.py +++ b/python/fate_flow/utils/__init__.py @@ -12,7 +12,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -from fate_flow.utils.base_utils import jprint - -__all__ = ["jprint"] diff --git a/python/fate_flow/utils/api_utils.py b/python/fate_flow/utils/api_utils.py index 2b9b99136..6c669c00d 100644 --- a/python/fate_flow/utils/api_utils.py +++ b/python/fate_flow/utils/api_utils.py @@ -13,53 +13,144 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import json import random import time from functools import wraps -from io import BytesIO -from flask import ( - Response, jsonify, send_file, - request as flask_request, -) -from werkzeug.http import HTTP_STATUS_CODES +import marshmallow +from flask import jsonify, send_file, request as flask_request -from fate_arch.common import ( - CoordinationCommunicationProtocol, CoordinationProxyService, - FederatedMode, -) -from fate_arch.common.base_utils import json_dumps, json_loads -from fate_arch.common.versions import get_fate_version +from webargs.flaskparser import parser -from fate_flow.db.job_default_config import JobDefaultConfig -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.db.service_registry import ServerRegistry -from fate_flow.entity import RetCode +from fate_flow.entity.types import CoordinationProxyService, CoordinationCommunicationProtocol, FederatedMode +from fate_flow.entity.code import ReturnCode +from fate_flow.errors import FateFlowError from fate_flow.hook import HookManager from fate_flow.hook.common.parameters import SignatureParameters -from fate_flow.settings import ( - API_VERSION, FATE_FLOW_SERVICE_NAME, HOST, HTTP_PORT, - PARTY_ID, PERMISSION_SWITCH, PROXY, PROXY_PROTOCOL, - REQUEST_MAX_WAIT_SEC, REQUEST_TRY_TIMES, REQUEST_WAIT_SEC, - SITE_AUTHENTICATION, stat_logger, -) -from fate_flow.utils.base_utils import compare_version -from fate_flow.utils.grpc_utils import ( - forward_grpc_packet, gen_routing_metadata, - get_command_federation_channel, wrap_grpc_packet, -) -from fate_flow.utils.log_utils import audit_logger, schedule_logger -from fate_flow.utils.permission_utils import get_permission_parameters +from fate_flow.runtime.job_default_config import JobDefaultConfig +from fate_flow.runtime.system_settings import PROXY_NAME, ENGINES, PROXY, HOST, HTTP_PORT, API_VERSION, \ + REQUEST_TRY_TIMES, REQUEST_MAX_WAIT_SEC, REQUEST_WAIT_SEC +from fate_flow.utils.log import getLogger +from fate_flow.utils.log_utils import schedule_logger, audit_logger from fate_flow.utils.requests_utils import request +parser.unknown = marshmallow.EXCLUDE + +stat_logger = getLogger() + + +class API: + class Input: + @staticmethod + def params(**kwargs): + return parser.use_kwargs(kwargs, location='querystring') + + @staticmethod + def form(**kwargs): + return parser.use_kwargs(kwargs, location='form') + + @staticmethod + def files(**kwargs): + return parser.use_kwargs(kwargs, location='files') + + @staticmethod + def json(**kwargs): + return parser.use_kwargs(kwargs, location='json') + + @staticmethod + def headers(**kwargs): + return parser.use_kwargs(kwargs, location="headers") + + class Output: + @staticmethod + def json(code=ReturnCode.Base.SUCCESS, message='success', data=None, job_id=None, **kwargs): + result_dict = { + "code": code, + "message": message, + "data": data, + "job_id": job_id, + } + + response = {} + for key, value in result_dict.items(): + if value is not None: + response[key] = value + # extra resp + for key, value in kwargs.items(): + response[key] = value + return jsonify(response) + + @staticmethod + def file(path_or_file, attachment_filename, as_attachment, mimetype="application/octet-stream"): + return send_file(path_or_file, download_name=attachment_filename, as_attachment=as_attachment, mimetype=mimetype) + + @staticmethod + def server_error_response(e): + if isinstance(e, FateFlowError): + return API.Output.json(code=e.code, message=e.message) + stat_logger.exception(e) + if len(e.args) > 1: + if isinstance(e.args[0], int): + return API.Output.json(code=e.args[0], message=e.args[1]) + else: + return API.Output.json(code=ReturnCode.Server.EXCEPTION, message=repr(e)) + return API.Output.json(code=ReturnCode.Server.EXCEPTION, message=repr(e)) + + @staticmethod + def args_error_response(e): + stat_logger.exception(e) + messages = e.data.get("messages", {}) + return API.Output.json(code=ReturnCode.API.INVALID_PARAMETER, message="Invalid request.", data=messages) + + @staticmethod + def fate_flow_exception(e: FateFlowError): + return API.Output.json(code=e.code, message=e.message) + + @staticmethod + def runtime_exception(code): + def _outer(func): + @wraps(func) + def _wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + if isinstance(e, FateFlowError): + raise e + else: + message = f"Request uri {flask_request.base_url} failed: {str(e)}" + return API.Output.json(code=code, message=message) + return _wrapper + return _outer + + +def get_federated_proxy_address(): + # protocol = CoordinationCommunicationProtocol.HTTP + proxy_name = PROXY_NAME + if ENGINES.get("federated_mode") == FederatedMode.SINGLE: + return HOST, HTTP_PORT, CoordinationCommunicationProtocol.HTTP, PROXY_NAME + if proxy_name == CoordinationProxyService.OSX: + host = PROXY.get(proxy_name).get("host") + port = PROXY.get(proxy_name).get("port") + proxy_name = CoordinationProxyService.ROLLSITE + protocol = CoordinationCommunicationProtocol.GRPC + + elif proxy_name == CoordinationProxyService.ROLLSITE: + host = PROXY.get(proxy_name).get("host") + port = PROXY.get(proxy_name).get("port") + protocol = CoordinationCommunicationProtocol.GRPC + + elif proxy_name == CoordinationProxyService.NGINX: + protocol = PROXY.get(proxy_name).get("protocol", "http") + host = PROXY.get(proxy_name).get(f"host") + port = PROXY.get(proxy_name).get(f"{protocol}_port") + else: + raise RuntimeError(f"Can not support coordinate proxy {proxy_name}， all proxy {PROXY.keys()}") + return host, port, protocol, proxy_name + -fate_version = get_fate_version() or '' -request_headers = { - 'User-Agent': f'{FATE_FLOW_SERVICE_NAME}/{fate_version}', - 'service': FATE_FLOW_SERVICE_NAME, - 'src_fate_ver': fate_version, -} +def generate_headers(party_id, body, initiator_party_id=""): + return HookManager.site_signature( + SignatureParameters(party_id=party_id, body=body, initiator_party_id=initiator_party_id)) def get_exponential_backoff_interval(retries, full_jitter=False): @@ -74,378 +165,21 @@ def get_exponential_backoff_interval(retries, full_jitter=False): return max(0, countdown) -def get_json_result(retcode=RetCode.SUCCESS, retmsg='success', data=None, job_id=None, meta=None): - result_dict = { - "retcode": retcode, - "retmsg": retmsg, - "data": data, - "jobId": job_id, - "meta": meta, - } - - response = {} - for key, value in result_dict.items(): - if value is not None: - response[key] = value - return jsonify(response) - - -def server_error_response(e): - stat_logger.exception(e) - - if len(e.args) > 1: - return get_json_result(retcode=RetCode.EXCEPTION_ERROR, retmsg=repr(e.args[0]), data=e.args[1]) - return get_json_result(retcode=RetCode.EXCEPTION_ERROR, retmsg=repr(e)) - - -def error_response(response_code, retmsg=None): - if retmsg is None: - retmsg = HTTP_STATUS_CODES.get(response_code, 'Unknown Error') - - return Response(json.dumps({ - 'retmsg': retmsg, - 'retcode': response_code, - }), status=response_code, mimetype='application/json') - - -def federated_api(job_id, method, endpoint, src_party_id, dest_party_id, src_role, json_body, federated_mode): - src_party_id = str(src_party_id or '') - dest_party_id = str(dest_party_id or '') - src_role = src_role or '' - - headers = request_headers.copy() - headers.update({ - 'src_party_id': src_party_id, - 'dest_party_id': dest_party_id, - 'src_role': src_role, - }) - - if SITE_AUTHENTICATION: - sign_obj = HookManager.site_signature(SignatureParameters(PARTY_ID, json_body)) - headers['site_signature'] = sign_obj.site_signature or '' - - kwargs = { - 'job_id': job_id, - 'method': method, - 'endpoint': endpoint, - 'src_party_id': src_party_id, - 'dest_party_id': dest_party_id, - 'src_role': src_role, - 'json_body': json_body, - 'headers': headers, - } - - if federated_mode == FederatedMode.SINGLE or kwargs['dest_party_id'] == '0': - kwargs.update({ - 'host': RuntimeConfig.JOB_SERVER_HOST, - 'port': RuntimeConfig.HTTP_PORT, - }) - - return federated_coordination_on_http(**kwargs) - - if federated_mode == FederatedMode.MULTIPLE: - host, port, protocol = get_federated_proxy_address(kwargs['src_party_id'], kwargs['dest_party_id']) - kwargs.update({ - 'host': host, - 'port': port, - }) - - if protocol == CoordinationCommunicationProtocol.HTTP: - return federated_coordination_on_http(**kwargs) - - if protocol == CoordinationCommunicationProtocol.GRPC: - return federated_coordination_on_grpc(**kwargs) - - raise Exception(f'{protocol} coordination communication protocol is not supported.') - - raise Exception(f'{federated_mode} work mode is not supported') - - -def local_api(job_id, method, endpoint, json_body): - return federated_api( - job_id=job_id, method=method, endpoint=endpoint, json_body=json_body, - src_party_id=PARTY_ID, dest_party_id=PARTY_ID, src_role='', - federated_mode=FederatedMode.SINGLE, - ) - - -def cluster_api(method, host, port, endpoint, json_body, headers=None): - return federated_coordination_on_http( - job_id='', method=method, host=host, port=port, endpoint=endpoint, - json_body=json_body, headers=headers or request_headers.copy(), - ) - - -def get_federated_proxy_address(src_party_id, dest_party_id): - src_party_id = str(src_party_id) - dest_party_id = str(dest_party_id) - - if PROXY_PROTOCOL == "default": - protocol = CoordinationCommunicationProtocol.HTTP - else: - protocol = PROXY_PROTOCOL - - if isinstance(PROXY, dict): - proxy_name = PROXY.get("name", CoordinationProxyService.FATEFLOW) - - if proxy_name == CoordinationProxyService.FATEFLOW and src_party_id == dest_party_id: - host = RuntimeConfig.JOB_SERVER_HOST - port = RuntimeConfig.HTTP_PORT - else: - host = PROXY["host"] - port = PROXY[f"{protocol}_port"] - - return ( - host, - port, - protocol, - ) - - if PROXY == CoordinationProxyService.ROLLSITE: - proxy_address = ServerRegistry.FATE_ON_EGGROLL[CoordinationProxyService.ROLLSITE] - - return ( - proxy_address["host"], - proxy_address.get("grpc_port", proxy_address["port"]), - CoordinationCommunicationProtocol.GRPC, - ) - - if PROXY == CoordinationProxyService.NGINX: - proxy_address = ServerRegistry.FATE_ON_SPARK[CoordinationProxyService.NGINX] - - return ( - proxy_address["host"], - proxy_address[f"{protocol}_port"], - protocol, - ) - - raise RuntimeError(f"can not support coordinate proxy {PROXY}") - - -def federated_coordination_on_http( - job_id, method, host, port, endpoint, - json_body, headers, **_, -): +def federated_coordination_on_http(method, host, port, endpoint, json_body, headers=None, params=None, + timeout=JobDefaultConfig.remote_request_timeout): url = f'http://{host}:{port}/{API_VERSION}{endpoint}' - - timeout = JobDefaultConfig.remote_request_timeout or 0 - timeout = timeout / 1000 or None - for t in range(REQUEST_TRY_TIMES): try: response = request( method=method, url=url, timeout=timeout, - headers=headers, json=json_body, + headers=headers, json=json_body, params=params ) response.raise_for_status() except Exception as e: - schedule_logger(job_id).warning(f'http api error: {url}\n{e}') + schedule_logger().warning(f'http api error: {url}\n{e}') if t >= REQUEST_TRY_TIMES - 1: raise e else: - audit_logger(job_id).info(f'http api response: {url}\n{response.text}') + audit_logger().info(f'http api response: {url}\n{response.text}') return response.json() - time.sleep(get_exponential_backoff_interval(t)) - - -def federated_coordination_on_grpc( - job_id, method, host, port, endpoint, - src_party_id, dest_party_id, - json_body, headers, **_, -): - endpoint = f"/{API_VERSION}{endpoint}" - timeout = JobDefaultConfig.remote_request_timeout or 0 - - _packet = wrap_grpc_packet( - json_body=json_body, http_method=method, url=endpoint, - src_party_id=src_party_id, dst_party_id=dest_party_id, - job_id=job_id, headers=headers, overall_timeout=timeout, - ) - _routing_metadata = gen_routing_metadata( - src_party_id=src_party_id, dest_party_id=dest_party_id, - ) - - for t in range(REQUEST_TRY_TIMES): - channel, stub = get_command_federation_channel(host, port) - - try: - _return, _call = stub.unaryCall.with_call( - _packet, metadata=_routing_metadata, - timeout=timeout / 1000 or None, - ) - except Exception as e: - schedule_logger(job_id).warning(f'grpc api error: {endpoint}\n{e}') - if t >= REQUEST_TRY_TIMES - 1: - raise e - else: - audit_logger(job_id).info(f'grpc api response: {endpoint}\n{_return}') - return json_loads(_return.body.value) - finally: - channel.close() - - time.sleep(get_exponential_backoff_interval(t)) - - -def proxy_api(role, _job_id, request_config): - headers = request_config.get('header', {}) - body = request_config.get('body', {}) - method = headers.get('METHOD', 'POST') - endpoint = headers.get('ENDPOINT', '') - job_id = headers.get('JOB-ID', _job_id) - src_party_id = headers.get('SRC-PARTY-ID', '') - dest_party_id = headers.get('DEST-PARTY-ID', '') - - _packet = forward_grpc_packet(body, method, endpoint, src_party_id, dest_party_id, role, job_id) - _routing_metadata = gen_routing_metadata(src_party_id, dest_party_id) - host, port, protocol = get_federated_proxy_address(src_party_id, dest_party_id) - - channel, stub = get_command_federation_channel(host, port) - _return, _call = stub.unaryCall.with_call(_packet, metadata=_routing_metadata) - channel.close() - - response = json_loads(_return.body.value) - return response - - -def forward_api(role, request_config): - role = role.upper() - if not hasattr(ServerRegistry, role): - ServerRegistry.load() - if not hasattr(ServerRegistry, role): - return {'retcode': 404, 'retmsg': f'role "{role.lower()}" not supported'} - registry = getattr(ServerRegistry, role) - - headers = request_config.get('header', {}) - body = request_config.get('body', {}) - method = headers.get('METHOD', 'POST') - endpoint = headers.get('ENDPOINT', '') - ip = registry.get('host', '') - port = registry.get('port', '') - url = f'http://{ip}:{port}{endpoint}' - audit_logger().info(f'api request: {url}') - - response = request(method=method, url=url, json=body, headers=headers) - response = ( - response.json() if response.status_code == 200 - else {'retcode': response.status_code, 'retmsg': response.text} - ) - audit_logger().info(response) - return response - - -def create_job_request_check(func): - @wraps(func) - def _wrapper(*_args, **_kwargs): - party_id = _kwargs.get("party_id") - role = _kwargs.get("role") - body = flask_request.json - headers = flask_request.headers - src_role = headers.get("scr_role") - src_party_id = headers.get("src_party_id") - - # permission check - if PERMISSION_SWITCH: - permission_return = HookManager.permission_check(get_permission_parameters(role, party_id, src_role, - src_party_id, body)) - if permission_return.code != RetCode.SUCCESS: - return get_json_result( - retcode=RetCode.PERMISSION_ERROR, - retmsg='permission check failed', - data=permission_return.to_dict() - ) - - # version check - src_fate_ver = headers.get('src_fate_ver') - if src_fate_ver is not None and compare_version(src_fate_ver, '1.7.0') == 'lt': - return get_json_result(retcode=RetCode.INCOMPATIBLE_FATE_VER, retmsg='Incompatible FATE versions', - data={'src_fate_ver': src_fate_ver, - "current_fate_ver": RuntimeConfig.get_env('FATE')}) - return func(*_args, **_kwargs) - return _wrapper - - -def validate_request(*args, **kwargs): - def wrapper(func): - @wraps(func) - def decorated_function(*_args, **_kwargs): - input_arguments = flask_request.json or flask_request.form.to_dict() - no_arguments = [] - error_arguments = [] - for arg in args: - if arg not in input_arguments: - no_arguments.append(arg) - for k, v in kwargs.items(): - config_value = input_arguments.get(k, None) - if config_value is None: - no_arguments.append(k) - elif isinstance(v, (tuple, list)): - if config_value not in v: - error_arguments.append((k, set(v))) - elif config_value != v: - error_arguments.append((k, v)) - if no_arguments or error_arguments: - error_string = "" - if no_arguments: - error_string += "required argument are missing: {}; ".format(",".join(no_arguments)) - if error_arguments: - error_string += "required argument values: {}".format(",".join(["{}={}".format(a[0], a[1]) for a in error_arguments])) - return get_json_result(retcode=RetCode.ARGUMENT_ERROR, retmsg=error_string) - return func(*_args, **_kwargs) - return decorated_function - return wrapper - - -def cluster_route(func): - @wraps(func) - def _route(*args, **kwargs): - request_data = flask_request.json or flask_request.form.to_dict() - - instance_id = request_data.get('instance_id') - if not instance_id: - return func(*args, **kwargs) - - request_data['forward_times'] = int(request_data.get('forward_times', 0)) + 1 - if request_data['forward_times'] > 2: - return error_response(429, 'Too many forwarding times.') - - instance = RuntimeConfig.SERVICE_DB.get_servers().get(instance_id) - if instance is None: - return error_response(404, 'Flow Instance not found.') - - if instance.http_address == f'{HOST}:{HTTP_PORT}': - return func(*args, **kwargs) - - endpoint = flask_request.full_path - prefix = f'/{API_VERSION}/' - if endpoint.startswith(prefix): - endpoint = endpoint[len(prefix) - 1:] - - response = cluster_api( - method=flask_request.method, - host=instance.host, - port=instance.http_port, - endpoint=endpoint, - json_body=request_data, - headers=flask_request.headers, - ) - return get_json_result(**response) - - return _route - - -def is_localhost(ip): - return ip in {'127.0.0.1', '::1', '[::1]', 'localhost'} - - -def send_file_in_mem(data, filename): - if not isinstance(data, (str, bytes)): - data = json_dumps(data) - if isinstance(data, str): - data = data.encode('utf-8') - - f = BytesIO() - f.write(data) - f.seek(0) - - return send_file(f, as_attachment=True, attachment_filename=filename) diff --git a/python/fate_flow/utils/base_utils.py b/python/fate_flow/utils/base_utils.py index 091d9e293..3d4926207 100644 --- a/python/fate_flow/utils/base_utils.py +++ b/python/fate_flow/utils/base_utils.py @@ -13,51 +13,160 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import datetime +import json import os +import random +import socket +import time +import uuid +from enum import Enum, IntEnum -from fate_arch.common.base_utils import json_dumps +class BaseType: + def to_dict(self): + return dict([(k.lstrip("_"), v) for k, v in self.__dict__.items()]) -FATE_FLOW_BASE = os.getenv("FATE_FLOW_BASE") + def to_dict_with_type(self): + def _dict(obj): + module = None + if issubclass(obj.__class__, BaseType): + data = {} + for attr, v in obj.__dict__.items(): + k = attr.lstrip("_") + data[k] = _dict(v) + module = obj.__module__ + elif isinstance(obj, (list, tuple)): + data = [] + for i, vv in enumerate(obj): + data.append(_dict(vv)) + elif isinstance(obj, dict): + data = {} + for _k, vv in obj.items(): + data[_k] = _dict(vv) + else: + data = obj + return {"type": obj.__class__.__name__, "data": data, "module": module} + return _dict(self) -def get_fate_flow_directory(*args): - global FATE_FLOW_BASE - if FATE_FLOW_BASE is None: - FATE_FLOW_BASE = os.path.abspath( - os.path.join( - os.path.dirname(os.path.realpath(__file__)), - os.pardir, - os.pardir, - os.pardir, - ) - ) - if args: - return os.path.join(FATE_FLOW_BASE, *args) - return FATE_FLOW_BASE +class CustomJSONEncoder(json.JSONEncoder): + def __init__(self, **kwargs): + self._with_type = kwargs.pop("with_type", False) + super().__init__(**kwargs) -def get_fate_flow_python_directory(*args): - return get_fate_flow_directory("python", *args) + def default(self, obj): + if isinstance(obj, datetime.datetime): + return obj.strftime("%Y-%m-%d %H:%M:%S") + elif isinstance(obj, datetime.date): + return obj.strftime("%Y-%m-%d") + elif isinstance(obj, datetime.timedelta): + return str(obj) + elif issubclass(type(obj), Enum) or issubclass(type(obj), IntEnum): + return obj.value + elif isinstance(obj, set): + return list(obj) + elif issubclass(type(obj), BaseType): + if not self._with_type: + return obj.to_dict() + else: + return obj.to_dict_with_type() + elif isinstance(obj, type): + return obj.__name__ + else: + return json.JSONEncoder.default(self, obj) -def jprint(src: dict, indent: int = 4): - print(json_dumps(src, indent=indent)) +def fate_uuid(): + return uuid.uuid1().hex -def compare_version(version: str, target_version: str): - ver_list = version.split('.') - tar_ver_list = target_version.split('.') - if int(ver_list[0]) >= int(tar_ver_list[0]): - if int(ver_list[1]) > int(tar_ver_list[1]): - return 'gt' - elif int(ver_list[1]) < int(tar_ver_list[1]): - return 'lt' - else: - if int(ver_list[2]) > int(tar_ver_list[2]): - return 'gt' - elif int(ver_list[2]) == int(tar_ver_list[2]): - return 'eq' - else: - return 'lt' - return 'lt' +def string_to_bytes(string): + return string if isinstance(string, bytes) else string.encode(encoding="utf-8") + + +def bytes_to_string(byte): + return byte.decode(encoding="utf-8") + + +def generate_random_id(length=6, only_number=False): + random_id = '' + if only_number: + chars = '0123456789' + else: + chars = 'AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz0123456789' + len_chars = len(chars) - 1 + for i in range(length): + random_id += chars[random.randint(0, len_chars)] + return random_id + + +def json_dumps(src, byte=False, indent=None, with_type=False): + dest = json.dumps(src, indent=indent, cls=CustomJSONEncoder, with_type=with_type) + if byte: + dest = string_to_bytes(dest) + return dest + + +def json_loads(src, object_hook=None, object_pairs_hook=None): + if isinstance(src, bytes): + src = bytes_to_string(src) + return json.loads(src, object_hook=object_hook, object_pairs_hook=object_pairs_hook) + + +def current_timestamp(): + return int(time.time() * 1000) + + +def timestamp_to_date(timestamp, format_string="%Y-%m-%d %H:%M:%S"): + if not timestamp: + timestamp = time.time() + timestamp = int(timestamp) / 1000 + time_array = time.localtime(timestamp) + str_date = time.strftime(format_string, time_array) + return str_date + + +def date_string_to_timestamp(time_str, format_string="%Y-%m-%d %H:%M:%S"): + time_array = time.strptime(time_str, format_string) + time_stamp = int(time.mktime(time_array) * 1000) + return time_stamp + + +def get_lan_ip(): + if os.name != "nt": + import fcntl + import struct + + def get_interface_ip(ifname): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + return socket.inet_ntoa( + fcntl.ioctl( + s.fileno(), + 0x8915, + struct.pack("256s", string_to_bytes(ifname[:15])), + )[20:24] + ) + + ip = socket.gethostbyname(socket.getfqdn()) + if ip.startswith("127.") and os.name != "nt": + interfaces = [ + "bond1", + "eth0", + "eth1", + "eth2", + "wlan0", + "wlan1", + "wifi0", + "ath0", + "ath1", + "ppp0", + ] + for ifname in interfaces: + try: + ip = get_interface_ip(ifname) + break + except IOError: + pass + return ip or "" diff --git a/python/fate_flow/utils/conf_utils.py b/python/fate_flow/utils/conf_utils.py new file mode 100644 index 000000000..318960d9d --- /dev/null +++ b/python/fate_flow/utils/conf_utils.py @@ -0,0 +1,48 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os + +from .file_utils import load_yaml_conf, get_fate_flow_directory + +SERVICE_CONF = "service_conf.yaml" +TRANSFER_CONF = "transfer_conf.yaml" + + +def conf_realpath(conf_name): + conf_path = f"conf/{conf_name}" + return os.path.join(get_fate_flow_directory(), conf_path) + + +def get_base_config(key, default=None, conf_name=SERVICE_CONF) -> dict: + local_config = {} + local_path = conf_realpath(f"local.{conf_name}") + + if os.path.exists(local_path): + local_config = load_yaml_conf(local_path) + if not isinstance(local_config, dict): + raise ValueError(f'Invalid config file: "{local_path}".') + + if key is not None and key in local_config: + return local_config[key] + + config_path = conf_realpath(conf_name) + config = load_yaml_conf(config_path) + + if not isinstance(config, dict): + raise ValueError(f'Invalid config file: "{config_path}".') + + config.update(local_config) + return config.get(key, default) if key is not None else config diff --git a/python/fate_flow/utils/config_adapter.py b/python/fate_flow/utils/config_adapter.py deleted file mode 100644 index 4098a0678..000000000 --- a/python/fate_flow/utils/config_adapter.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from copy import deepcopy - -from fate_flow.entity import RunParameters - - -class JobRuntimeConfigAdapter(object): - def __init__(self, job_runtime_conf): - job_runtime_conf = deepcopy(job_runtime_conf) - if 'job_parameters' not in job_runtime_conf: - job_runtime_conf['job_parameters'] = {} - if 'common' not in job_runtime_conf['job_parameters']: - job_runtime_conf['job_parameters']['common'] = {} - self.job_runtime_conf = job_runtime_conf - - def get_common_parameters(self): - if int(self.job_runtime_conf.get('dsl_version', 1)) == 2: - job_parameters = RunParameters(**self.job_runtime_conf.get("job_parameters", {}).get("common", {})) - self.job_runtime_conf['job_parameters']['common'] = job_parameters.to_dict() - else: - if "processors_per_node" in self.job_runtime_conf['job_parameters']: - self.job_runtime_conf['job_parameters']["eggroll_run"] = \ - {"eggroll.session.processors.per.node": self.job_runtime_conf['job_parameters']["processors_per_node"]} - job_parameters = RunParameters(**self.job_runtime_conf['job_parameters']) - self.job_runtime_conf['job_parameters'] = job_parameters.to_dict() - return job_parameters - - def update_common_parameters(self, common_parameters: RunParameters): - if int(self.job_runtime_conf.get("dsl_version", 1)) == 2: - self.job_runtime_conf["job_parameters"]["common"] = common_parameters.to_dict() - else: - self.job_runtime_conf["job_parameters"] = common_parameters.to_dict() - return self.job_runtime_conf - - def get_job_parameters_dict(self, job_parameters: RunParameters = None): - if job_parameters: - if int(self.job_runtime_conf.get('dsl_version', 1)) == 2: - self.job_runtime_conf['job_parameters']['common'] = job_parameters.to_dict() - else: - self.job_runtime_conf['job_parameters'] = job_parameters.to_dict() - return self.job_runtime_conf['job_parameters'] - - def check_removed_parameter(self): - check_list = [] - if self.check_backend(): - check_list.append("backend") - if self.check_work_mode(): - check_list.append("work_mode") - return ','.join(check_list) - - def check_backend(self): - if int(self.job_runtime_conf.get('dsl_version', 1)) == 2: - backend = self.job_runtime_conf['job_parameters'].get('common', {}).get('backend') - else: - backend = self.job_runtime_conf['job_parameters'].get('backend') - return backend is not None - - def check_work_mode(self): - if int(self.job_runtime_conf.get('dsl_version', 1)) == 2: - work_mode = self.job_runtime_conf['job_parameters'].get('common', {}).get('work_mode') - else: - work_mode = self.job_runtime_conf['job_parameters'].get('work_mode') - return work_mode is not None - - def get_job_type(self): - if int(self.job_runtime_conf.get('dsl_version', 1)) == 2: - job_type = self.job_runtime_conf['job_parameters'].get('common', {}).get('job_type') - if not job_type: - job_type = self.job_runtime_conf['job_parameters'].get('job_type', 'train') - else: - job_type = self.job_runtime_conf['job_parameters'].get('job_type', 'train') - return job_type - - def update_model_id_version(self, model_id=None, model_version=None): - if int(self.job_runtime_conf.get('dsl_version', 1)) == 2: - if model_id: - self.job_runtime_conf['job_parameters']['common']['model_id'] = model_id - if model_version: - self.job_runtime_conf['job_parameters']['common']['model_version'] = model_version - else: - if model_id: - self.job_runtime_conf['job_parameters']['model_id'] = model_id - if model_version: - self.job_runtime_conf['job_parameters']['model_version'] = model_version - return self.job_runtime_conf diff --git a/python/fate_flow/utils/data_utils.py b/python/fate_flow/utils/data_utils.py deleted file mode 100644 index d37ef1e79..000000000 --- a/python/fate_flow/utils/data_utils.py +++ /dev/null @@ -1,166 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_arch.abc import StorageTableMetaABC, AddressABC -from fate_arch.common.address import MysqlAddress, HiveAddress -from fate_arch.common.data_utils import default_output_fs_path -from fate_arch.computing import ComputingEngine -from fate_arch.storage import StorageEngine, StorageTableMeta -from fate_flow.entity.types import InputSearchType -from fate_arch import storage - - -def get_header_schema(header_line, id_delimiter, extend_sid=False): - header_source_item = header_line.split(id_delimiter) - if extend_sid: - header = id_delimiter.join(header_source_item).strip() - sid = get_extend_id_name() - else: - header = id_delimiter.join(header_source_item[1:]).strip() - sid = header_source_item[0].strip() - return {'header': header, 'sid': sid} - - -def get_extend_id_name(): - return "extend_sid" - - -def get_sid_data_line(values, id_delimiter, fate_uuid, line_index, **kwargs): - return line_extend_uuid(fate_uuid, line_index), list_to_str(values, id_delimiter=id_delimiter) - - -def line_extend_uuid(fate_uuid, line_index): - return fate_uuid + str(line_index) - - -def get_auto_increasing_sid_data_line(values, id_delimiter, line_index, **kwargs): - return line_index, list_to_str(values, id_delimiter=id_delimiter) - - -def get_data_line(values, id_delimiter, **kwargs): - return values[0], list_to_str(values[1:], id_delimiter=id_delimiter) - - -def list_to_str(input_list, id_delimiter): - return id_delimiter.join(list(map(str, input_list))) - - -def convert_output( - input_name, - input_namespace, - output_name, - output_namespace, - computing_engine: ComputingEngine = ComputingEngine.EGGROLL, - output_storage_address={}, - ) -> (StorageTableMetaABC, AddressABC, StorageEngine): - input_table_meta = StorageTableMeta(name=input_name, namespace=input_namespace) - - if not input_table_meta: - raise RuntimeError( - f"can not found table name: {input_name} namespace: {input_namespace}" - ) - address_dict = output_storage_address.copy() - if input_table_meta.get_engine() in [StorageEngine.PATH]: - from fate_arch.storage import PathStoreType - - address_dict["name"] = output_name - address_dict["namespace"] = output_namespace - address_dict["storage_type"] = PathStoreType.PICTURE - address_dict["path"] = input_table_meta.get_address().path - output_table_address = StorageTableMeta.create_address( - storage_engine=StorageEngine.PATH, address_dict=address_dict - ) - output_table_engine = StorageEngine.PATH - elif computing_engine == ComputingEngine.STANDALONE: - from fate_arch.storage import StandaloneStoreType - - address_dict["name"] = output_name - address_dict["namespace"] = output_namespace - address_dict["storage_type"] = StandaloneStoreType.ROLLPAIR_LMDB - output_table_address = StorageTableMeta.create_address( - storage_engine=StorageEngine.STANDALONE, address_dict=address_dict - ) - output_table_engine = StorageEngine.STANDALONE - elif computing_engine == ComputingEngine.EGGROLL: - from fate_arch.storage import EggRollStoreType - - address_dict["name"] = output_name - address_dict["namespace"] = output_namespace - address_dict["storage_type"] = EggRollStoreType.ROLLPAIR_LMDB - output_table_address = StorageTableMeta.create_address( - storage_engine=StorageEngine.EGGROLL, address_dict=address_dict - ) - output_table_engine = StorageEngine.EGGROLL - elif computing_engine == ComputingEngine.SPARK: - if input_table_meta.get_engine() == StorageEngine.HIVE: - output_table_address = input_table_meta.get_address() - output_table_address.name = output_name - output_table_engine = input_table_meta.get_engine() - elif input_table_meta.get_engine() == StorageEngine.LOCALFS: - output_table_address = input_table_meta.get_address() - output_table_address.path = default_output_fs_path( - name=output_name, - namespace=output_namespace, - storage_engine=StorageEngine.LOCALFS - ) - output_table_engine = input_table_meta.get_engine() - else: - address_dict["path"] = default_output_fs_path( - name=output_name, - namespace=output_namespace, - prefix=address_dict.get("path_prefix"), - storage_engine=StorageEngine.HDFS - ) - output_table_address = StorageTableMeta.create_address( - storage_engine=StorageEngine.HDFS, address_dict=address_dict - ) - output_table_engine = StorageEngine.HDFS - elif computing_engine == ComputingEngine.LINKIS_SPARK: - output_table_address = input_table_meta.get_address() - output_table_address.name = output_name - output_table_engine = input_table_meta.get_engine() - else: - raise RuntimeError(f"can not support computing engine {computing_engine}") - return input_table_meta, output_table_address, output_table_engine - - -def get_input_data_min_partitions(input_data, role, party_id): - min_partition = None - if role != 'arbiter': - for data_type, data_location in input_data[role][party_id].items(): - table_info = {'name': data_location.split('.')[1], 'namespace': data_location.split('.')[0]} - table_meta = storage.StorageTableMeta(name=table_info['name'], namespace=table_info['namespace']) - if table_meta: - table_partition = table_meta.get_partitions() - if not min_partition or min_partition > table_partition: - min_partition = table_partition - return min_partition - - -def get_input_search_type(parameters): - if "name" in parameters and "namespace" in parameters: - return InputSearchType.TABLE_INFO - elif "job_id" in parameters and "component_name" in parameters and "data_name" in parameters: - return InputSearchType.JOB_COMPONENT_OUTPUT - else: - return InputSearchType.UNKNOWN - - -def address_filter(address): - if isinstance(address, MysqlAddress): - address.passwd = None - if isinstance(address, HiveAddress): - address.password = None - return address.__dict__ diff --git a/python/fate_flow/utils/db_utils.py b/python/fate_flow/utils/db_utils.py new file mode 100644 index 000000000..e8caecc6c --- /dev/null +++ b/python/fate_flow/utils/db_utils.py @@ -0,0 +1,20 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +def get_dynamic_db_model(base, job_id): + return type(base.model(table_index=get_dynamic_tracking_table_index(job_id=job_id))) + + +def get_dynamic_tracking_table_index(job_id): + return job_id[:8] \ No newline at end of file diff --git a/python/fate_flow/utils/deepspeed_utils.py b/python/fate_flow/utils/deepspeed_utils.py deleted file mode 100644 index 6438fa81d..000000000 --- a/python/fate_flow/utils/deepspeed_utils.py +++ /dev/null @@ -1,71 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging - -from fate_arch.common.log import getLogger -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.entity.run_status import TaskStatus -from fate_flow.scheduling_apps.client import ControllerClient -from fate_flow.utils.log_utils import schedule_logger -from fate_flow.worker.base_worker import BaseWorker - - -class Submit(BaseWorker): - def _run(self): - try: - from eggroll.deepspeed.submit import client - client = client.DeepspeedJob(session_id=self.args.session_id) - config = self.args.config - schedule_logger(self.args.job_id).info(f"start submit deepspeed task {self.args.session_id}") - schedule_logger(self.args.job_id).info(f"submit config {config}") - client.submit( - world_size=config.get("world_size"), - command_arguments=config.get("command_arguments"), - environment_variables=config.get("environment_variables"), - files=config.get("files"), - resource_options=config.get("resource_options"), - options=config.get("options") - ) - schedule_logger(self.args.job_id).info(f"submit deepspeed task success") - - logger = getLogger() - threads = [] - for handle in logger.handlers: - handle.setFormatter(logging.Formatter("%(message)s")) - for _type in ["DEBUG", "INFO", "ERROR"]: - threads.extend( - client.write_logs_to(log_type=_type, logging=getattr(logger, _type.lower())) - ) - for thread in threads: - thread.join() - except Exception as e: - task_info = { - "job_id": self.args.job_id, - "role": self.args.role, - "party_id": self.args.party_id, - "task_id": self.args.task_id, - "task_version": self.args.task_version, - "component_name": self.args.component_name, - "party_status": TaskStatus.FAILED, - } - - RuntimeConfig.init_config(JOB_SERVER_HOST=self.args.job_server.split(':')[0], HTTP_PORT=self.args.job_server.split(':')[1]) - ControllerClient.report_task(task_info) - schedule_logger(self.args.job_id).exception(e) - - -if __name__ == "__main__": - Submit().run() diff --git a/python/fate_flow/utils/detect_utils.py b/python/fate_flow/utils/detect_utils.py deleted file mode 100644 index 0154615f9..000000000 --- a/python/fate_flow/utils/detect_utils.py +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import typing - - -def check_config(config: typing.Dict, required_arguments: typing.List): - if not config or not isinstance(config, dict): - raise TypeError('no parameters') - - no_arguments = [] - error_arguments = [] - for require_argument in required_arguments: - if isinstance(require_argument, tuple): - config_value = config.get(require_argument[0], None) - if isinstance(require_argument[1], (tuple, list)): - if config_value not in require_argument[1]: - error_arguments.append(require_argument) - elif config_value != require_argument[1]: - error_arguments.append(require_argument) - elif require_argument not in config: - no_arguments.append(require_argument) - - if no_arguments or error_arguments: - error_string = "" - if no_arguments: - error_string += "required parameters are missing: {}; ".format(",".join(no_arguments)) - if error_arguments: - error_string += "required parameter values: {}".format(",".join(["{}={}".format(a[0], a[1]) for a in error_arguments])) - raise KeyError(error_string) diff --git a/python/fate_flow/utils/dsl_exception.py b/python/fate_flow/utils/dsl_exception.py deleted file mode 100644 index 8810a1762..000000000 --- a/python/fate_flow/utils/dsl_exception.py +++ /dev/null @@ -1,218 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -class BaseDSLException(Exception): - def __init__(self, msg): - self.msg = msg - - -class DSLNotExistError(BaseDSLException): - def __str__(self): - return "There are no dsl, please check if the role and party id are correct" - - -class SubmitConfNotExistError(Exception): - def __str__(self): - return "SubMitConf is None, does not exist" - - -class ComponentFieldNotExistError(Exception): - def __str__(self): - return "No components filed in dsl, please have a check!" - - -class ModuleException(Exception): - def __init__(self, component=None, module=None, input=None, output_model=None, - output_data=None, other_info=None, value_type=None): - self.component = component - self.module = module - self.input = input - self.output_data = output_data - self.output_model = output_model - self.other_info = other_info - self.value_type = value_type - - -class ComponentNotExistError(ModuleException): - def __str__(self): - return "Component {} does not exist, please have a check".format(self.component) - - -class ModuleFieldNotExistError(ModuleException): - def __str__(self): - return "Component {}, module field does not exist in dsl, please have a check".format(self.component) - - -class ModuleNotExistError(ModuleException): - def __str__(self): - return "Component {}, module {} does not exist under the fold federatedml.setting_conf".format(self.component, - self.module) - - -class ModuleConfigError(ModuleException): - def __str__(self): - return "Component {}, module {} config error, message is {}".format(self.component, self.module, - self.other_info[0]) - - -class DataNotExistInSubmitConfError(BaseDSLException): - def __str__(self): - return "{} does not exist in submit conf's data, please check!".format(self.msg) - - -class DefaultRuntimeConfNotExistError(ModuleException): - def __str__(self): - return "Default runtime conf of component {}, module {}, does not exist".format(self.component, self.module) - - -class DefaultRuntimeConfNotJsonError(ModuleException): - def __str__(self): - return "Default runtime conf of component {}, module {} should be json format file, but error occur: {}".format(self.component, self.module, self.other_info) - - -class InputComponentNotExistError(ModuleException): - def __str__(self): - return "Component {}'s {} input {} does not exist".format(self.component, self.value_type, self.input) - - -class InputNameNotExistError(ModuleException): - def __str__(self): - return "Component {}' s {} input {}'s output {} does not exist".format(self.component, - self.input, - self.value_type, - self.other_info) - - -class ComponentInputTypeError(ModuleException): - def __str__(self): - return "Input of component {} should be dict".format(self.component) - - -class ComponentOutputTypeError(ModuleException): - def __str__(self): - return "Output of component {} should be dict, but {} does not match".format(self.component, self.other_info) - - -class ComponentInputDataTypeError(ModuleException): - def __str__(self): - return "Component {}'s input data type should be dict".format(self.component) - - -class ComponentInputValueTypeError(ModuleException): - def __str__(self): - return "Component {}'s input {} type should be list, but {} not match".format(self.component, - self.value_type, - self.other_info) - - -class ComponentOutputKeyTypeError(ModuleException): - def __str__(self): - return "Component {}'s output key {} value type should be list".format(self.component, - self.other_info) - - -class ParameterException(Exception): - def __init__(self, parameter, role=None, msg=None): - self.parameter = parameter - self.role = role - self.msg = msg - - -class ParamClassNotExistError(ModuleException): - def __str__(self): - return "Component {}, module {}'s param class {} does not exist".format(self.component, self.module, - self.other_info) - - -class RoleParameterNotListError(ParameterException): - def __str__(self): - return "Role {} role parameter {} should be list".format(self.role, self.parameter) - - -class RoleParameterNotConsistencyError(ParameterException): - def __str__(self): - return "Role {} role parameter {} should be a list of same length with roles".format(self.role, self.parameter) - - -class ParameterCheckError(ModuleException): - def __str__(self): - return "Component {}, module {}, does not pass component check, error msg is {}".format(self.component, - self.module, - self.other_info) - - -class RedundantParameterError(ParameterCheckError): - def __str__(self): - return "Component {}, module {}, has redundant parameter {}".format(self.component, - self.module, - self.other_info) - - -class ComponentDuplicateError(ModuleException): - def __str__(self): - return "Component {} is duplicated, running before".format(self.component) - - -class DegreeNotZeroError(ModuleException): - def __str__(self): - return "Component {}' in degree should be zero for topological sort".format(self.component) - - -class ModeError(BaseDSLException): - def __str__(self): - return "dsl' s mode should be train or predict" - - -class LoopError(Exception): - def __init__(self, components=None): - self.components = components - - def __str__(self): - if self.components is not None: - return "{} form a loop".format("->".join(self.components)) - else: - return "component relationship forms a dependency loop" - - -class NamingError(ModuleException): - def __str__(self): - return "Component's name should be format of name_index, index is start from 0 " + \ - "and be consecutive for same module, {} is error".format(self.component) - - -class NamingIndexError(ModuleException): - def __str__(self): - return "Component {}'s index should be an integer start from 0".format(self.component) - - -class NamingFormatError(ModuleException): - def __str__(self): - return "Component name {}'is not illegal, it should be consits of letter, digit, '-' and '_'".format(self.component) - - -class ComponentMultiMappingError(ModuleException): - def __str__(self): - return "Component prefix {} should be used for only one module, but another".format(self.component) - - -class DeployComponentNotExistError(BaseDSLException): - def __init__(self, msg): - self.msg = msg - - def __str__(self): - return "components {} not exist in training dsl, can not deploy!!!".format(self.msg) - diff --git a/python/fate_flow/utils/engine_utils.py b/python/fate_flow/utils/engine_utils.py new file mode 100644 index 000000000..9b962ea0d --- /dev/null +++ b/python/fate_flow/utils/engine_utils.py @@ -0,0 +1,110 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from fate_flow.engine.relation_ship import Relationship +from fate_flow.entity.types import EngineType, FederationEngine, StorageEngine, ComputingEngine, FederatedMode +from fate_flow.utils import conf_utils + + +def get_engine_class_members(engine_class) -> list: + members = [] + for k, v in engine_class.__dict__.items(): + if k in ["__module__", "__dict__", "__weakref__", "__doc__"]: + continue + members.append(v) + return members + + +def get_engines(): + engines = { + EngineType.COMPUTING: None, + EngineType.FEDERATION: None, + EngineType.STORAGE: None, + } + + # check service_conf.yaml + if ( + conf_utils.get_base_config("default_engines", {}).get(EngineType.COMPUTING) + is None + ): + raise RuntimeError(f"must set default_engines on conf/service_conf.yaml") + default_engines = conf_utils.get_base_config("default_engines") + + # computing engine + if default_engines.get(EngineType.COMPUTING) is None: + raise RuntimeError( + f"{EngineType.COMPUTING} is None," + f"Please check default_engines on conf/service_conf.yaml" + ) + engines[EngineType.COMPUTING] = default_engines[EngineType.COMPUTING].lower() + if engines[EngineType.COMPUTING] not in get_engine_class_members(ComputingEngine): + raise RuntimeError(f"{engines[EngineType.COMPUTING]} is illegal") + + # federation engine + if default_engines.get(EngineType.FEDERATION) is not None: + engines[EngineType.FEDERATION] = default_engines[EngineType.FEDERATION].lower() + + # storage engine + if default_engines.get(EngineType.STORAGE) is not None: + engines[EngineType.STORAGE] = default_engines[EngineType.STORAGE].lower() + + # set default storage engine and federation engine by computing engine + for t in (EngineType.STORAGE, EngineType.FEDERATION): + if engines.get(t) is None: + # use default relation engine + engines[t] = Relationship.Computing[engines[EngineType.COMPUTING]][t][ + "default" + ] + + # set default federated mode by federation engine + if engines[EngineType.FEDERATION] == FederationEngine.STANDALONE: + engines["federated_mode"] = FederatedMode.SINGLE + else: + engines["federated_mode"] = FederatedMode.MULTIPLE + + if engines[EngineType.STORAGE] not in get_engine_class_members(StorageEngine): + raise RuntimeError(f"{engines[EngineType.STORAGE]} is illegal") + + if engines[EngineType.FEDERATION] not in get_engine_class_members(FederationEngine): + raise RuntimeError(f"{engines[EngineType.FEDERATION]} is illegal") + + for t in [EngineType.FEDERATION]: + if ( + engines[t] + not in Relationship.Computing[engines[EngineType.COMPUTING]][t]["support"] + ): + raise RuntimeError( + f"{engines[t]} is not supported in {engines[EngineType.COMPUTING]}" + ) + + return engines + + +def is_standalone(): + return ( + get_engines().get(EngineType.FEDERATION).lower() == FederationEngine.STANDALONE + ) + + +def get_engines_config_from_conf(group_map=False): + engines_config = {} + for engine_type in { + EngineType.COMPUTING, + EngineType.FEDERATION, + EngineType.STORAGE, + }: + engines_config[engine_type] = {} + for _name, _conf in conf_utils.get_base_config(engine_type, {}).items(): + engines_config[engine_type][_name] = _conf + return engines_config \ No newline at end of file diff --git a/python/fate_flow/utils/file_utils.py b/python/fate_flow/utils/file_utils.py new file mode 100644 index 000000000..413ed3b2e --- /dev/null +++ b/python/fate_flow/utils/file_utils.py @@ -0,0 +1,98 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import os + +from ruamel import yaml + +from fate_flow.runtime.env import is_in_virtualenv + +PROJECT_BASE = os.getenv("FATE_PROJECT_BASE") +FATE_PYTHON_PATH = os.getenv("FATE_PYTHONPATH") + + +def get_project_base_directory(*args): + global PROJECT_BASE + if PROJECT_BASE is None: + PROJECT_BASE = os.path.abspath( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + os.pardir, + os.pardir, + os.pardir, + ) + ) + if args: + return os.path.join(PROJECT_BASE, *args) + return PROJECT_BASE + + +def get_fate_python_path(): + global FATE_PYTHON_PATH + if not FATE_PYTHON_PATH: + FATE_PYTHON_PATH = get_project_base_directory("fate", "python") + if not os.path.exists(FATE_PYTHON_PATH): + FATE_PYTHON_PATH = get_project_base_directory("python") + if not os.path.exists(FATE_PYTHON_PATH): + return + return FATE_PYTHON_PATH + + +def get_fate_flow_directory(*args): + if is_in_virtualenv(): + fate_flow_dir = os.path.abspath( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + os.pardir + ) + ) + else: + fate_flow_dir = os.path.abspath( + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + os.pardir, + os.pardir, + os.pardir, + ) + ) + if args: + return os.path.join(fate_flow_dir, *args) + return fate_flow_dir + + +def load_yaml_conf(conf_path): + if not os.path.isabs(conf_path): + conf_path = os.path.join(get_fate_flow_directory(), conf_path) + try: + with open(conf_path) as f: + return yaml.safe_load(f) + except Exception as e: + raise EnvironmentError( + "loading yaml file config from {} failed:".format(conf_path), e + ) + + +def rewrite_json_file(filepath, json_data): + with open(filepath, "w") as f: + json.dump(json_data, f, indent=4, separators=(",", ": ")) + f.close() + + +def save_file(file, path): + with open(path, 'wb') as f: + content = file.stream.read() + f.write(content) \ No newline at end of file diff --git a/python/fate_flow/utils/grpc_utils.py b/python/fate_flow/utils/grpc_utils.py index 2c754f2ed..a331ccef9 100644 --- a/python/fate_flow/utils/grpc_utils.py +++ b/python/fate_flow/utils/grpc_utils.py @@ -13,24 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import grpc +from fate_flow.errors.server_error import ResponseException +from fate_flow.proto.rollsite import proxy_pb2_grpc, basic_meta_pb2, proxy_pb2 -from fate_arch.protobuf.python import basic_meta_pb2, proxy_pb2, proxy_pb2_grpc -from fate_arch.common.base_utils import json_dumps, json_loads - -from fate_flow.db.job_default_config import JobDefaultConfig -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.settings import FATE_FLOW_SERVICE_NAME, GRPC_OPTIONS, GRPC_PORT, HOST +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.runtime.system_settings import FATE_FLOW_SERVICE_NAME, GRPC_PORT, HOST, REMOTE_REQUEST_TIMEOUT +from fate_flow.utils.base_utils import json_loads, json_dumps from fate_flow.utils.log_utils import audit_logger from fate_flow.utils.requests_utils import request -def get_command_federation_channel(host, port): - channel = grpc.insecure_channel(f"{host}:{port}", GRPC_OPTIONS) - stub = proxy_pb2_grpc.DataTransferServiceStub(channel) - return channel, stub - - def gen_routing_metadata(src_party_id, dest_party_id): routing_head = ( ("service", "fateflow"), @@ -42,8 +34,8 @@ def gen_routing_metadata(src_party_id, dest_party_id): return routing_head -def wrap_grpc_packet(json_body, http_method, url, src_party_id, dst_party_id, job_id=None, headers=None, overall_timeout=None): - overall_timeout = JobDefaultConfig.remote_request_timeout if overall_timeout is None else overall_timeout +def wrap_grpc_packet(json_body, http_method, url, src_party_id, dst_party_id, job_id=None, headers=None, + overall_timeout=REMOTE_REQUEST_TIMEOUT): _src_end_point = basic_meta_pb2.Endpoint(ip=HOST, port=GRPC_PORT) _src = proxy_pb2.Topic(name=job_id, partyId="{}".format(src_party_id), role=FATE_FLOW_SERVICE_NAME, callback=_src_end_point) _dst = proxy_pb2.Topic(name=job_id, partyId="{}".format(dst_party_id), role=FATE_FLOW_SERVICE_NAME, callback=None) @@ -61,8 +53,7 @@ def get_url(_suffix): class UnaryService(proxy_pb2_grpc.DataTransferServiceServicer): - @staticmethod - def unaryCall(_request, context): + def unaryCall(self, _request, context): packet = _request header = packet.header _suffix = packet.body.key @@ -85,20 +76,16 @@ def unaryCall(_request, context): audit_logger(job_id).info('rpc receive: {}'.format(packet)) audit_logger(job_id).info("rpc receive: {} {}".format(get_url(_suffix), param)) resp = request(method=method, url=get_url(_suffix), json=param_dict, headers=headers) - resp_json = resp.json() - + audit_logger(job_id).info(f"resp: {resp.text}") + resp_json = response_json(resp) return wrap_grpc_packet(resp_json, method, _suffix, dst.partyId, src.partyId, job_id) -def forward_grpc_packet(_json_body, _method, _url, _src_party_id, _dst_party_id, role, job_id=None, - overall_timeout=None): - overall_timeout = JobDefaultConfig.remote_request_timeout if overall_timeout is None else overall_timeout - _src_end_point = basic_meta_pb2.Endpoint(ip=HOST, port=GRPC_PORT) - _src = proxy_pb2.Topic(name=job_id, partyId="{}".format(_src_party_id), role=FATE_FLOW_SERVICE_NAME, callback=_src_end_point) - _dst = proxy_pb2.Topic(name=job_id, partyId="{}".format(_dst_party_id), role=role, callback=None) - _task = proxy_pb2.Task(taskId=job_id) - _command = proxy_pb2.Command(name=_url) - _conf = proxy_pb2.Conf(overallTimeout=overall_timeout) - _meta = proxy_pb2.Metadata(src=_src, dst=_dst, task=_task, command=_command, operator=_method, conf=_conf) - _data = proxy_pb2.Data(key=_url, value=bytes(json_dumps(_json_body), 'utf-8')) - return proxy_pb2.Packet(header=_meta, body=_data) +def response_json(response): + try: + return response.json() + except: + audit_logger().exception(response.text) + e = ResponseException(response=response.text) + return {"code": e.code, "message": e.message} + diff --git a/python/fate_flow/utils/io_utils.py b/python/fate_flow/utils/io_utils.py new file mode 100644 index 000000000..9f0f3bf2e --- /dev/null +++ b/python/fate_flow/utils/io_utils.py @@ -0,0 +1,208 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib +import re +from abc import ABCMeta +from dataclasses import dataclass +from typing import Optional + +# see https://www.rfc-editor.org/rfc/rfc3986#appendix-B +# scheme = $2 +# authority = $4 +# path = $5 +# query = $7 +# fragment = $9 +from fate_flow.runtime.system_settings import STANDALONE_DATA_HOME + +_uri_regex = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") + + +@dataclass +class URI: + schema: str + path: str + query: Optional[str] = None + fragment: Optional[str] = None + authority: Optional[str] = None + + @classmethod + def from_string(cls, uri: str) -> "URI": + match = _uri_regex.fullmatch(uri) + if match is None: + raise ValueError(f"`{uri}` is not valid uri") + _, schema, _, authority, path, _, query, _, fragment = match.groups() + return URI(schema, path, query, fragment, authority) + + def to_schema(self): + for cls in ConcrateURI.__subclasses__(): + if cls.schema() == self.schema: + return cls.from_uri(self) + raise NotImplementedError(f"uri schema `{self.schema}` not found") + + +class ConcrateURI(metaclass=ABCMeta): + @classmethod + def schema(cls) -> str: + ... + + @classmethod + def from_uri(cls, uri: URI) -> "ConcrateURI": + ... + + def create_file(self, name): + ... + + def to_string(self): + ... + + +_EGGROLL_NAME_MAX_SIZE = 128 + + +@dataclass +class FileURI(ConcrateURI): + path: str + + @classmethod + def schema(cls): + return "file" + + @classmethod + def from_uri(cls, uri: URI): + return FileURI(uri.path) + + def create_file(self, name): + return FileURI(f"{self.path}/{name}") + + def to_string(self): + return f"file://{self.path}" + + +@dataclass +class HttpURI(ConcrateURI): + path: str + + @classmethod + def schema(cls): + return "http" + + @classmethod + def from_uri(cls, uri: URI): + return HttpURI(uri.path) + + def create_file(self, name): + return HttpURI(path=f"{self.path}/{name}") + + def to_string(self): + return f"{self.path}" + + +@dataclass +class EggrollURI(ConcrateURI): + namespace: str + name: str + + @classmethod + def schema(cls): + return "eggroll" + + @classmethod + def from_uri(cls, uri: URI): + _, namespace, *names = uri.path.split("/") + name = "_".join(names) + if len(name) > _EGGROLL_NAME_MAX_SIZE: + name = hashlib.md5(name.encode(encoding="utf8")).hexdigest()[:_EGGROLL_NAME_MAX_SIZE] + return EggrollURI(namespace, name) + + def create_file(self, name): + name = f"{self.name}_{name}" + if len(name) > _EGGROLL_NAME_MAX_SIZE: + name = hashlib.md5(name.encode(encoding="utf8")).hexdigest()[:_EGGROLL_NAME_MAX_SIZE] + return EggrollURI(namespace=self.namespace, name=name) + + def to_string(self): + return f"eggroll:///{self.namespace}/{self.name}" + + +@dataclass +class StandaloneURI(ConcrateURI): + namespace: str + name: str + + @classmethod + def schema(cls): + return "standalone" + + @classmethod + def from_uri(cls, uri: URI): + if STANDALONE_DATA_HOME in uri.path: + _, namespace, *names = uri.path.split(STANDALONE_DATA_HOME)[1].split("/") + else: + _, namespace, *names = uri.path.split("/") + name = "_".join(names) + if len(name) > _EGGROLL_NAME_MAX_SIZE: + name = hashlib.md5(name.encode(encoding="utf8")).hexdigest()[:_EGGROLL_NAME_MAX_SIZE] + return StandaloneURI(namespace, name) + + def create_file(self, name): + name = f"{self.name}_{name}" + if len(name) > _EGGROLL_NAME_MAX_SIZE: + name = hashlib.md5(name.encode(encoding="utf8")).hexdigest()[:_EGGROLL_NAME_MAX_SIZE] + return StandaloneURI(namespace=self.namespace, name=name) + + def to_string(self): + return f"standalone:///{self.namespace}/{self.name}" + + +@dataclass +class HdfsURI(ConcrateURI): + path: str + authority: Optional[str] = None + + @classmethod + def schema(cls): + return "hdfs" + + @classmethod + def from_uri(cls, uri: URI): + return HdfsURI(uri.path, uri.authority) + + def create_file(self, name): + return HdfsURI(path=f"{self.path}/{name}", authority=self.authority) + + def to_string(self): + if self.authority: + return f"hdfs://{self.authority}{self.path}" + else: + return f"hdfs://{self.path}" + + +@dataclass +class LocalfsURI(ConcrateURI): + path: str + + @classmethod + def schema(cls): + return "path" + + @classmethod + def from_uri(cls, uri: URI): + return LocalfsURI(uri.path) + + def create_file(self, name): + return LocalfsURI(path=f"{self.path}/{name}") + + def to_string(self): + return f"{self.path}" diff --git a/python/fate_flow/utils/job_utils.py b/python/fate_flow/utils/job_utils.py index cc8eda27f..70382d2c0 100644 --- a/python/fate_flow/utils/job_utils.py +++ b/python/fate_flow/utils/job_utils.py @@ -13,35 +13,20 @@ # limitations under the License. # import datetime -import errno import os -import random -import sys import threading -import typing -from functools import wraps - -from fate_arch.common import FederatedMode, file_utils -from fate_arch.common.base_utils import current_timestamp, fate_uuid, json_dumps - -from fate_flow.db.db_models import DB, Job, Task -from fate_flow.db.db_utils import query_db -from fate_flow.db.job_default_config import JobDefaultConfig -from fate_flow.db.service_registry import ServerRegistry -from fate_flow.entity import JobConfiguration, RunParameters -from fate_flow.entity.run_status import JobStatus, TaskStatus -from fate_flow.entity.types import InputSearchType -from fate_flow.settings import FATE_BOARD_DASHBOARD_ENDPOINT -from fate_flow.utils import data_utils, detect_utils, process_utils, session_utils, schedule_utils -from fate_flow.utils.base_utils import get_fate_flow_directory -from fate_flow.utils.log_utils import schedule_logger -from fate_flow.utils.schedule_utils import get_dsl_parser_by_version +import yaml -PIPELINE_COMPONENT_NAME = 'pipeline' -PIPELINE_MODEL_ALIAS = 'pipeline' -PIPELINE_COMPONENT_MODULE_NAME = 'Pipeline' -PIPELINE_MODEL_NAME = 'Pipeline' +from fate_flow.db.base_models import DB +from fate_flow.db.db_models import Job, Task +from fate_flow.entity.spec.dag import InheritConfSpec +from fate_flow.entity.types import TaskStatus +from fate_flow.errors.server_error import InheritanceFailed +from fate_flow.manager.operation.job_saver import JobSaver +from fate_flow.runtime.system_settings import LOG_DIR, JOB_DIR, WORKERS_DIR +from fate_flow.utils.base_utils import fate_uuid, current_timestamp +from fate_flow.utils.log_utils import schedule_logger class JobIdGenerator(object): @@ -56,7 +41,6 @@ def next_id(self): """ generate next job id with locking """ - #todo: there is duplication in the case of multiple instances deployment now = datetime.datetime.now() with JobIdGenerator._lock: if self._pre_timestamp == now: @@ -79,6 +63,10 @@ def generate_job_id(): return job_id_generator.next_id() +def generate_deepspeed_id(task_id): + return f"deepspeed_{task_id}" + + def generate_task_id(job_id, component_name): return '{}_{}'.format(job_id, component_name) @@ -88,7 +76,7 @@ def generate_task_version_id(task_id, task_version): def generate_session_id(task_id, task_version, role, party_id, suffix=None, random_end=False): - items = [task_id, str(task_version), role, str(party_id)] + items = [task_id, str(task_version), role, party_id] if suffix: items.append(suffix) if random_end: @@ -96,433 +84,122 @@ def generate_session_id(task_id, task_version, role, party_id, suffix=None, rand return "_".join(items) -def generate_task_input_data_namespace(task_id, task_version, role, party_id): - return "input_data_{}".format(generate_session_id(task_id=task_id, - task_version=task_version, - role=role, - party_id=party_id)) - - def get_job_directory(job_id, *args): - return os.path.join(get_fate_flow_directory(), 'jobs', job_id, *args) + return os.path.join(JOB_DIR, job_id, *args) def get_job_log_directory(job_id, *args): - return os.path.join(get_fate_flow_directory(), 'logs', job_id, *args) - - -def get_task_directory(job_id, role, party_id, component_name, task_id, task_version, **kwargs): - return get_job_directory(job_id, role, party_id, component_name, task_id, task_version) - - -def get_general_worker_directory(worker_name, worker_id, *args): - return os.path.join(get_fate_flow_directory(), worker_name, worker_id, *args) - - -def get_general_worker_log_directory(worker_name, worker_id, *args): - return os.path.join(get_fate_flow_directory(), 'logs', worker_name, worker_id, *args) + return os.path.join(LOG_DIR, job_id, *args) -def check_config(config: typing.Dict, required_parameters: typing.List): - for parameter in required_parameters: - if parameter not in config: - return False, 'configuration no {} parameter'.format(parameter) - else: - return True, 'ok' - - -def check_job_conf(runtime_conf, job_dsl): - detect_utils.check_config(runtime_conf, ['initiator', 'role']) - detect_utils.check_config(runtime_conf['initiator'], ['role', 'party_id']) - # deal party id - runtime_conf['initiator']['party_id'] = int(runtime_conf['initiator']['party_id']) - for r in runtime_conf['role'].keys(): - for i in range(len(runtime_conf['role'][r])): - runtime_conf['role'][r][i] = int(runtime_conf['role'][r][i]) - constraint_check(runtime_conf, job_dsl) - - -def runtime_conf_basic(if_local=False): - job_runtime_conf = { - "dsl_version": 2, - "initiator": {}, - "job_parameters": { - "common": { - "federated_mode": FederatedMode.SINGLE - }, - }, - "role": {}, - "component_parameters": {} - } - if if_local: - job_runtime_conf["initiator"]["role"] = "local" - job_runtime_conf["initiator"]["party_id"] = 0 - job_runtime_conf["role"]["local"] = [0] - return job_runtime_conf - - -def new_runtime_conf(job_dir, method, module, role, party_id): - if role: - conf_path_dir = os.path.join(job_dir, method, module, role, str(party_id)) +def get_task_directory(job_id, role, party_id, task_name, task_version, input=False, output=False, base_dir="", **kwargs): + if not base_dir: + base_path = get_job_directory(job_id) else: - conf_path_dir = os.path.join(job_dir, method, module, str(party_id)) - os.makedirs(conf_path_dir, exist_ok=True) - return os.path.join(conf_path_dir, 'runtime_conf.json') - - -def save_job_conf(job_id, role, party_id, dsl, runtime_conf, runtime_conf_on_party, train_runtime_conf, pipeline_dsl=None): - path_dict = get_job_conf_path(job_id=job_id, role=role, party_id=party_id) - dump_job_conf(path_dict=path_dict, - dsl=dsl, - runtime_conf=runtime_conf, - runtime_conf_on_party=runtime_conf_on_party, - train_runtime_conf=train_runtime_conf, - pipeline_dsl=pipeline_dsl) - return path_dict - - -def save_task_using_job_conf(task: Task): - task_dir = get_task_directory(job_id=task.f_job_id, - role=task.f_role, - party_id=task.f_party_id, - component_name=task.f_component_name, - task_id=task.f_task_id, - task_version=str(task.f_task_version)) - return save_using_job_conf(task.f_job_id, task.f_role, task.f_party_id, config_dir=task_dir) - - -def save_using_job_conf(job_id, role, party_id, config_dir): - path_dict = get_job_conf_path(job_id=job_id, role=role, party_id=party_id, specified_dir=config_dir) - job_configuration = get_job_configuration(job_id=job_id, - role=role, - party_id=party_id) - dump_job_conf(path_dict=path_dict, - dsl=job_configuration.dsl, - runtime_conf=job_configuration.runtime_conf, - runtime_conf_on_party=job_configuration.runtime_conf_on_party, - train_runtime_conf=job_configuration.train_runtime_conf, - pipeline_dsl=None) - return path_dict - - -def dump_job_conf(path_dict, dsl, runtime_conf, runtime_conf_on_party, train_runtime_conf, pipeline_dsl=None): - os.makedirs(os.path.dirname(path_dict.get('dsl_path')), exist_ok=True) - os.makedirs(os.path.dirname(path_dict.get('runtime_conf_on_party_path')), exist_ok=True) - for data, conf_path in [(dsl, path_dict['dsl_path']), - (runtime_conf, path_dict['runtime_conf_path']), - (runtime_conf_on_party, path_dict['runtime_conf_on_party_path']), - (train_runtime_conf, path_dict['train_runtime_conf_path']), - (pipeline_dsl, path_dict['pipeline_dsl_path'])]: - with open(conf_path, 'w+') as f: - f.truncate() - if not data: - data = {} - f.write(json_dumps(data, indent=4)) - f.flush() - return path_dict - - -@DB.connection_context() -def get_job_configuration(job_id, role, party_id) -> JobConfiguration: - jobs = Job.select(Job.f_dsl, Job.f_runtime_conf, Job.f_train_runtime_conf, Job.f_runtime_conf_on_party).where(Job.f_job_id == job_id, - Job.f_role == role, - Job.f_party_id == party_id) - if jobs: - job = jobs[0] - return JobConfiguration(**job.to_human_model_dict()) - + base_path = f"{base_dir}/{job_id}" -def get_task_using_job_conf(task_info: dict): - task_dir = get_task_directory(**task_info) - return read_job_conf(task_info["job_id"], task_info["role"], task_info["party_id"], task_dir) + if input: + return os.path.join(base_path, role, party_id, task_name, str(task_version), "input") + if output: + return os.path.join(base_path, role, party_id, task_name, str(task_version), "output") + else: + return os.path.join(base_path, role, party_id, task_name, str(task_version)) -def read_job_conf(job_id, role, party_id, specified_dir=None): - path_dict = get_job_conf_path(job_id=job_id, role=role, party_id=party_id, specified_dir=specified_dir) - conf_dict = {} - for key, path in path_dict.items(): - config = file_utils.load_json_conf(path) - conf_dict[key.rstrip("_path")] = config - return JobConfiguration(**conf_dict) +def get_general_worker_directory(worker_name, worker_id, *args): + return os.path.join(WORKERS_DIR, worker_name, worker_id, *args) -def get_job_conf_path(job_id, role, party_id, specified_dir=None): - conf_dir = get_job_directory(job_id) if not specified_dir else specified_dir - job_dsl_path = os.path.join(conf_dir, 'job_dsl.json') - job_runtime_conf_path = os.path.join(conf_dir, 'job_runtime_conf.json') - if not specified_dir: - job_runtime_conf_on_party_path = os.path.join(conf_dir, role, str(party_id), 'job_runtime_on_party_conf.json') - else: - job_runtime_conf_on_party_path = os.path.join(conf_dir, 'job_runtime_on_party_conf.json') - train_runtime_conf_path = os.path.join(conf_dir, 'train_runtime_conf.json') - pipeline_dsl_path = os.path.join(conf_dir, 'pipeline_dsl.json') - return {'dsl_path': job_dsl_path, - 'runtime_conf_path': job_runtime_conf_path, - 'runtime_conf_on_party_path': job_runtime_conf_on_party_path, - 'train_runtime_conf_path': train_runtime_conf_path, - 'pipeline_dsl_path': pipeline_dsl_path} +def get_general_worker_log_directory(worker_name, worker_id, *args): + return os.path.join(LOG_DIR, worker_name, worker_id, *args) -@DB.connection_context() -def get_upload_job_configuration_summary(upload_tasks: typing.List[Task]): - jobs_run_conf = {} - for task in upload_tasks: - jobs = Job.select(Job.f_job_id, Job.f_runtime_conf_on_party, Job.f_description).where(Job.f_job_id == task.f_job_id) - job = jobs[0] - jobs_run_conf[job.f_job_id] = job.f_runtime_conf_on_party["component_parameters"]["role"]["local"]["0"]["upload_0"] - jobs_run_conf[job.f_job_id]["notes"] = job.f_description - return jobs_run_conf +def generate_model_info(job_id): + model_id = job_id + model_version = "0" + return model_id, model_version @DB.connection_context() -def get_job_parameters(job_id, role, party_id): - jobs = Job.select(Job.f_runtime_conf_on_party).where(Job.f_job_id == job_id, - Job.f_role == role, - Job.f_party_id == party_id) +def get_job_resource_info(job_id, role, party_id): + jobs = Job.select(Job.f_cores, Job.f_memory).where( + Job.f_job_id == job_id, + Job.f_role == role, + Job.f_party_id == party_id) if jobs: job = jobs[0] - return job.f_runtime_conf_on_party.get("job_parameters") + return job.f_cores, job.f_memory else: - return {} + return None, None @DB.connection_context() -def get_job_dsl(job_id, role, party_id): - jobs = Job.select(Job.f_dsl).where(Job.f_job_id == job_id, - Job.f_role == role, - Job.f_party_id == party_id) - if jobs: - job = jobs[0] - return job.f_dsl +def get_task_resource_info(job_id, role, party_id, task_id, task_version): + tasks = Task.select(Task.f_task_cores, Task.f_memory, Task.f_launcher_name).where( + Task.f_job_id == job_id, + Task.f_role == role, + Task.f_party_id == party_id, + Task.f_task_id == task_id, + Task.f_task_version == task_version + ) + if tasks: + task = tasks[0] + return task.f_task_cores, task.f_memory, task.f_launcher_name else: - return {} + return None, None -@DB.connection_context() -def list_job(limit=0, offset=0, query=None, order_by=None): - return query_db(Job, limit, offset, query, order_by) - +def save_job_dag(job_id, dag): + job_conf_file = os.path.join(JOB_DIR, job_id, "dag.yaml") + os.makedirs(os.path.dirname(job_conf_file), exist_ok=True) + with open(job_conf_file, "w") as f: + f.write(yaml.dump(dag)) -@DB.connection_context() -def list_task(limit=0, offset=0, query=None, order_by=None): - return query_db(Task, limit, offset, query, order_by) +def inheritance_check(inheritance: InheritConfSpec = None): + if not inheritance: + return + if not inheritance.task_list: + raise InheritanceFailed( + task_list=inheritance.task_list, + position="dag_schema.dag.conf.inheritance.task_list" + ) + inheritance_jobs = JobSaver.query_job(job_id=inheritance.job_id) + inheritance_tasks = JobSaver.query_task(job_id=inheritance.job_id) + if not inheritance_jobs: + raise InheritanceFailed(job_id=inheritance.job_id, detail=f"no found job {inheritance.job_id}") + task_status = {} + for task in inheritance_tasks: + task_status[task.f_task_name] = task.f_status -def check_job_process(pid): - if pid < 0: - return False - if pid == 0: - raise ValueError('invalid PID 0') - try: - os.kill(pid, 0) - except OSError as err: - if err.errno == errno.ESRCH: - # ESRCH == No such process - return False - elif err.errno == errno.EPERM: - # EPERM clearly means there's a process to deny access to - return True - else: - # According to "man 2 kill" possible error values are - # (EINVAL, EPERM, ESRCH) - raise - else: - return True + for task_name in inheritance.task_list: + if task_name not in task_status.keys(): + raise InheritanceFailed(job_id=inheritance.job_id, task_name=task_name, detail="no found task name") + elif task_status[task_name] not in [TaskStatus.SUCCESS, TaskStatus.PASS]: + raise InheritanceFailed( + job_id=inheritance.job_id, + task_name=task_name, + task_status=task_status[task_name], + detail=f"task status need in [{TaskStatus.SUCCESS}, {TaskStatus.PASS}]" + ) -def check_job_is_timeout(job: Job): - job_parameters = job.f_runtime_conf_on_party["job_parameters"] - timeout = job_parameters.get("timeout", JobDefaultConfig.job_timeout) +def check_task_is_timeout(task: Task): now_time = current_timestamp() - running_time = (now_time - job.f_create_time)/1000 - if running_time > timeout: - schedule_logger(job.f_job_id).info(f'run time {running_time}s timeout') + if not task.f_start_time: + return False + running_time = (now_time - task.f_start_time)/1000 + if task.f_timeout and running_time > task.f_timeout: + schedule_logger(task.f_job_id).info(f'task {task.f_task_name} run time {running_time}s timeout') + schedule_logger(task.f_job_id).error(f'task {task.f_task_name} timeout[{task.f_timeout}s]') return True else: return False -def start_session_stop(task): - job_parameters = RunParameters(**get_job_parameters(job_id=task.f_job_id, role=task.f_role, party_id=task.f_party_id)) - session_manager_id = generate_session_id(task.f_task_id, task.f_task_version, task.f_role, task.f_party_id) - if task.f_status != TaskStatus.WAITING: - schedule_logger(task.f_job_id).info(f'start run subprocess to stop task sessions {session_manager_id}') - else: - schedule_logger(task.f_job_id).info(f'task is waiting, pass stop sessions {session_manager_id}') - return - task_dir = os.path.join(get_job_directory(job_id=task.f_job_id), task.f_role, - task.f_party_id, task.f_component_name, 'session_stop') - os.makedirs(task_dir, exist_ok=True) - process_cmd = [ - sys.executable or 'python3', - sys.modules[session_utils.SessionStop.__module__].__file__, - '--session', session_manager_id, - '--computing', job_parameters.computing_engine, - '--federation', job_parameters.federation_engine, - '--storage', job_parameters.storage_engine, - '-c', 'stop' if task.f_status == JobStatus.SUCCESS else 'kill' - ] - p = process_utils.run_subprocess(job_id=task.f_job_id, config_dir=task_dir, process_cmd=process_cmd) - p.wait() - p.poll() - - -def get_timeout(job_id, timeout, runtime_conf, dsl): - try: - if timeout > 0: - schedule_logger(job_id).info(f'setting job timeout {timeout}') - return timeout - else: - default_timeout = job_default_timeout(runtime_conf, dsl) - schedule_logger(job_id).info(f'setting job timeout {timeout} not a positive number, using the default timeout {default_timeout}') - return default_timeout - except: - default_timeout = job_default_timeout(runtime_conf, dsl) - schedule_logger(job_id).info(f'setting job timeout {timeout} is incorrect, using the default timeout {default_timeout}') - return default_timeout - - -def job_default_timeout(runtime_conf, dsl): - # future versions will improve - timeout = JobDefaultConfig.job_timeout - return timeout - - -def get_board_url(job_id, role, party_id): - board_url = "http://{}:{}{}".format( - ServerRegistry.FATEBOARD.get("host"), - ServerRegistry.FATEBOARD.get("port"), - FATE_BOARD_DASHBOARD_ENDPOINT).format(job_id, role, party_id) - return board_url - - -def check_job_inheritance_parameters(job, inheritance_jobs, inheritance_tasks): - if not inheritance_jobs: - raise Exception( - f"no found job {job.f_inheritance_info.get('job_id')} role {job.f_role} party id {job.f_party_id}") - inheritance_job = inheritance_jobs[0] - task_status = {} - for task in inheritance_tasks: - task_status[task.f_component_name] = task.f_status - for component in job.f_inheritance_info.get('component_list'): - if component not in task_status.keys(): - raise Exception(f"job {job.f_inheritance_info.get('job_id')} no found component {component}") - elif task_status[component] not in [TaskStatus.SUCCESS, TaskStatus.PASS]: - raise Exception(F"job {job.f_inheritance_info.get('job_id')} component {component} status:{task_status[component]}") - dsl_parser = get_dsl_parser_by_version() - dsl_parser.verify_conf_reusability(inheritance_job.f_runtime_conf, job.f_runtime_conf, job.f_inheritance_info.get('component_list')) - dsl_parser.verify_dsl_reusability(inheritance_job.f_dsl, job.f_dsl, job.f_inheritance_info.get('component_list', [])) - - -def get_job_all_components(dsl): - return [dsl['components'][component_name]['module'].lower() for component_name in dsl['components'].keys()] - - -def constraint_check(job_runtime_conf, job_dsl): - if job_dsl: - all_components = get_job_all_components(job_dsl) - glm = ['heterolr', 'heterolinr', 'heteropoisson'] - for cpn in glm: - if cpn in all_components: - roles = job_runtime_conf.get('role') - if 'guest' in roles.keys() and 'arbiter' in roles.keys() and 'host' in roles.keys(): - for party_id in set(roles['guest']) & set(roles['arbiter']): - if party_id not in roles['host'] or len(set(roles['guest']) & set(roles['arbiter'])) != len(roles['host']): - raise Exception("{} component constraint party id, please check role config:{}".format(cpn, job_runtime_conf.get('role'))) - - -def get_job_dataset(is_initiator, role, party_id, roles, job_args): - dataset = {} - dsl_version = 1 - if job_args.get('dsl_version'): - if job_args.get('dsl_version') == 2: - dsl_version = 2 - for _role, _role_party_args in job_args.items(): - if _role == "dsl_version": - continue - if is_initiator or _role == role: - for _party_index in range(len(_role_party_args)): - _party_id = roles[_role][_party_index] - if is_initiator or _party_id == party_id: - dataset[_role] = dataset.get(_role, {}) - dataset[_role][_party_id] = dataset[_role].get( - _party_id, {}) - if dsl_version == 1: - for _data_type, _data_location in _role_party_args[_party_index]['args']['data'].items(): - dataset[_role][_party_id][_data_type] = '{}.{}'.format( - _data_location['namespace'], _data_location['name']) - else: - for key in _role_party_args[_party_index].keys(): - for _data_type, _data_location in _role_party_args[_party_index][key].items(): - search_type = data_utils.get_input_search_type(parameters=_data_location) - if search_type is InputSearchType.TABLE_INFO: - dataset[_role][_party_id][key] = '{}.{}'.format(_data_location['namespace'], _data_location['name']) - elif search_type is InputSearchType.JOB_COMPONENT_OUTPUT: - dataset[_role][_party_id][key] = '{}.{}.{}'.format(_data_location['job_id'], _data_location['component_name'], _data_location['data_name']) - else: - dataset[_role][_party_id][key] = "unknown" - return dataset - - -def asynchronous_function(func): - @wraps(func) - def _wrapper(*args, **kwargs): - is_asynchronous = kwargs.pop("is_asynchronous", False) - if is_asynchronous: - thread = threading.Thread(target=func, args=args, kwargs=kwargs) - thread.start() - is_asynchronous = True - return is_asynchronous - else: - return func(*args, **kwargs) - return _wrapper - - -def task_report(jobs, tasks): - job = jobs[0] - report_list = [] - dsl_parser = schedule_utils.get_job_dsl_parser( - dsl=job.f_dsl, - runtime_conf=job.f_runtime_conf, - train_runtime_conf=job.f_train_runtime_conf - ) - name_component_maps, hierarchical_structure = dsl_parser.get_dsl_hierarchical_structure() - for index, cpn_list in enumerate(hierarchical_structure): - for name in cpn_list: - for task in tasks: - if task.f_component_name == name: - report_list.append({ - "component_name": task.f_component_name, "start_time": task.f_start_time, - "end_time": task.f_end_time, "elapsed": task.f_elapsed, "status": task.f_status, - "index": index - }) - return report_list - - -def get_component_parameters(job_providers, dsl_parser, provider_detail, role, party_id): - component_parameters = dict() - for component in job_providers.keys(): - provider_info = job_providers[component]["provider"] - provider_name = provider_info["name"] - provider_version = provider_info["version"] - parameter = dsl_parser.parse_component_parameters(component, - provider_detail, - provider_name, - provider_version, - local_role=role, - local_party_id=party_id) - module_name = dsl_parser.get_component_info(component_name=component).get_module().lower() - if module_name not in component_parameters.keys(): - component_parameters[module_name] = [parameter.get("ComponentParam", {})] - else: - component_parameters[module_name].append(parameter.get("ComponentParam", {})) - return component_parameters - - -def generate_retry_interval(cur_retry, max_retry_cnt, long_retry_cnt): - - if cur_retry < max_retry_cnt - long_retry_cnt: - retry_interval = random.random() * 10 + 5 - else: - retry_interval = round(300 + random.random() * 10, 3) - return retry_interval +def check_party_in(role, party_id, parties): + for party in parties: + if party.role == role: + if party_id in party.party_id: + return True + return False diff --git a/python/fate_flow/utils/log.py b/python/fate_flow/utils/log.py new file mode 100644 index 000000000..e8abfe320 --- /dev/null +++ b/python/fate_flow/utils/log.py @@ -0,0 +1,213 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import inspect +import traceback +import logging +import os +from logging.handlers import TimedRotatingFileHandler +from threading import RLock + +from fate_flow.runtime.system_settings import LOG_SHARE + + +class LoggerFactory(object): + TYPE = "FILE" + LOG_FORMAT = "[%(levelname)s] [%(asctime)s] [jobId] [%(process)s:%(thread)s] - [%(module)s.%(funcName)s] [line:%(lineno)d]: %(message)s" + LEVEL = logging.DEBUG + logger_dict = {} + global_handler_dict = {} + + LOG_DIR = None + PARENT_LOG_DIR = None + log_share = LOG_SHARE + + append_to_parent_log = None + + lock = RLock() + # CRITICAL = 50 + # FATAL = CRITICAL + # ERROR = 40 + # WARNING = 30 + # WARN = WARNING + # INFO = 20 + # DEBUG = 10 + # NOTSET = 0 + levels = (10, 20, 30, 40) + schedule_logger_dict = {} + + @staticmethod + def set_directory(directory, parent_log_dir=None, append_to_parent_log=None, force=False): + if parent_log_dir: + LoggerFactory.PARENT_LOG_DIR = parent_log_dir + if append_to_parent_log: + LoggerFactory.append_to_parent_log = append_to_parent_log + with LoggerFactory.lock: + if not LoggerFactory.LOG_DIR or force: + LoggerFactory.LOG_DIR = directory + if LoggerFactory.log_share: + oldmask = os.umask(000) + os.makedirs(LoggerFactory.LOG_DIR, exist_ok=True) + os.umask(oldmask) + else: + os.makedirs(LoggerFactory.LOG_DIR, exist_ok=True) + for loggerName, ghandler in LoggerFactory.global_handler_dict.items(): + for className, (logger, handler) in LoggerFactory.logger_dict.items(): + logger.removeHandler(ghandler) + ghandler.close() + LoggerFactory.global_handler_dict = {} + for className, (logger, handler) in LoggerFactory.logger_dict.items(): + logger.removeHandler(handler) + _handler = None + if handler: + handler.close() + if className != "default": + _handler = LoggerFactory.get_handler(className) + logger.addHandler(_handler) + LoggerFactory.assemble_global_handler(logger) + LoggerFactory.logger_dict[className] = logger, _handler + + @staticmethod + def new_logger(name): + logger = logging.getLogger(name) + logger.propagate = False + logger.setLevel(LoggerFactory.LEVEL) + return logger + + @staticmethod + def get_logger(class_name=None): + with LoggerFactory.lock: + if class_name in LoggerFactory.logger_dict.keys(): + logger, handler = LoggerFactory.logger_dict[class_name] + if not logger: + logger, handler = LoggerFactory.init_logger(class_name) + else: + logger, handler = LoggerFactory.init_logger(class_name) + return logger + + @staticmethod + def get_global_handler(logger_name, level=None, log_dir=None): + if not LoggerFactory.LOG_DIR: + return logging.StreamHandler() + if log_dir: + logger_name_key = logger_name + "_" + log_dir + else: + logger_name_key = logger_name + "_" + LoggerFactory.LOG_DIR + # if loggerName not in LoggerFactory.globalHandlerDict: + if logger_name_key not in LoggerFactory.global_handler_dict: + with LoggerFactory.lock: + if logger_name_key not in LoggerFactory.global_handler_dict: + handler = LoggerFactory.get_handler(logger_name, level, log_dir) + LoggerFactory.global_handler_dict[logger_name_key] = handler + return LoggerFactory.global_handler_dict[logger_name_key] + + @staticmethod + def get_handler(class_name, level=None, log_dir=None, log_type=None, job_id=None): + if not log_type: + if not LoggerFactory.LOG_DIR or not class_name: + return logging.StreamHandler() + + if not log_dir: + log_file = os.path.join(LoggerFactory.LOG_DIR, "{}.log".format(class_name)) + else: + log_file = os.path.join(log_dir, "{}.log".format(class_name)) + else: + log_file = os.path.join(log_dir, "fate_flow_{}.log".format( + log_type) if level == LoggerFactory.LEVEL else 'fate_flow_{}_error.log'.format(log_type)) + job_id = job_id or os.getenv("FATE_JOB_ID") + if job_id: + formatter = logging.Formatter(LoggerFactory.LOG_FORMAT.replace("jobId", job_id)) + else: + formatter = logging.Formatter(LoggerFactory.LOG_FORMAT.replace("jobId", "Server")) + os.makedirs(os.path.dirname(log_file), exist_ok=True) + if LoggerFactory.log_share: + handler = ROpenHandler(log_file, + when='D', + interval=1, + backupCount=14, + delay=True) + else: + handler = TimedRotatingFileHandler(log_file, + when='D', + interval=1, + backupCount=14, + delay=True) + + if level: + handler.level = level + + handler.setFormatter(formatter) + return handler + + @staticmethod + def init_logger(class_name): + with LoggerFactory.lock: + logger = LoggerFactory.new_logger(class_name) + handler = None + if class_name: + handler = LoggerFactory.get_handler(class_name) + logger.addHandler(handler) + LoggerFactory.logger_dict[class_name] = logger, handler + + else: + LoggerFactory.logger_dict["default"] = logger, handler + + LoggerFactory.assemble_global_handler(logger) + return logger, handler + + @staticmethod + def assemble_global_handler(logger): + if isinstance(LoggerFactory.LEVEL, str): + LoggerFactory.LEVEL = logging._nameToLevel[LoggerFactory.LEVEL] + if LoggerFactory.LOG_DIR: + for level in LoggerFactory.levels: + if level >= LoggerFactory.LEVEL: + level_logger_name = logging._levelToName[level] + logger.addHandler(LoggerFactory.get_global_handler(level_logger_name, level)) + if LoggerFactory.append_to_parent_log and LoggerFactory.PARENT_LOG_DIR: + for level in LoggerFactory.levels: + if level >= LoggerFactory.LEVEL: + level_logger_name = logging._levelToName[level] + logger.addHandler( + LoggerFactory.get_global_handler(level_logger_name, level, LoggerFactory.PARENT_LOG_DIR)) + + +def setDirectory(directory=None): + LoggerFactory.set_directory(directory) + + +def setLevel(level): + LoggerFactory.LEVEL = level + + +def getLogger(className=None, useLevelFile=False): + if className is None: + frame = inspect.stack()[1] + module = inspect.getmodule(frame[0]) + className = 'stat' + return LoggerFactory.get_logger(className) + + +def exception_to_trace_string(ex): + return "".join(traceback.TracebackException.from_exception(ex).format()) + + +class ROpenHandler(TimedRotatingFileHandler): + def _open(self): + prevumask = os.umask(000) + rtv = TimedRotatingFileHandler._open(self) + os.umask(prevumask) + return rtv diff --git a/python/fate_flow/utils/log_sharing_utils.py b/python/fate_flow/utils/log_sharing_utils.py deleted file mode 100644 index 4c750b97a..000000000 --- a/python/fate_flow/utils/log_sharing_utils.py +++ /dev/null @@ -1,92 +0,0 @@ -import os -import subprocess - -from fate_flow.utils.base_utils import get_fate_flow_directory -from fate_flow.utils.log_utils import replace_ip - -JOB = ["jobSchedule", "jobScheduleError"] -PARTY = ["partyError", "partyWarning", "partyInfo", "partyDebug"] -COMPONENT = ["componentInfo"] -LOGMapping = { - "jobSchedule": "fate_flow_schedule.log", - "jobScheduleError": "fate_flow_schedule_error.log", - "partyError": "ERROR.log", - "partyWarning": "WARNING.log", - "partyInfo": "INFO.log", - "partyDebug": "DEBUG.log", - "componentInfo": "INFO.log" -} - - -def parameters_check(log_type, job_id, role, party_id, component_name): - if log_type in JOB: - if not job_id: - return False - if log_type in PARTY: - if not job_id or not role or not party_id: - return False - if log_type in COMPONENT: - if not job_id or not role or not party_id or not component_name: - return False - return True - - -class LogCollector(): - def __init__(self, log_type, job_id, party_id="", role="", component_name="", **kwargs): - self.log_type = log_type - self.job_id = job_id - self.party_id = str(party_id) - self.role = role - self.component_name = component_name - - def get_log_file_path(self): - status = parameters_check(self.log_type, self.job_id, self.role, self.party_id, self.component_name) - if not status: - raise Exception(f"job type {self.log_type} Missing parameters") - type_dict = { - "jobSchedule": os.path.join(self.job_id, "fate_flow_schedule.log"), - "jobScheduleError": os.path.join(self.job_id, "fate_flow_schedule_error.log"), - "partyError": os.path.join(self.job_id, self.role, self.party_id, "ERROR.log"), - "partyWarning": os.path.join(self.job_id, self.role, self.party_id, "WARNING.log"), - "partyInfo": os.path.join(self.job_id,self.role, self.party_id, "INFO.log"), - "partyDebug": os.path.join(self.job_id, self.role, self.party_id, "DEBUG.log"), - "componentInfo": os.path.join(self.job_id, self.role, self.party_id, self.component_name, "INFO.log") - } - if self.log_type not in type_dict.keys(): - raise Exception(f"no found log type {self.log_type}") - return os.path.join(get_fate_flow_directory('logs'), type_dict[self.log_type]) - - def cat_log(self, begin, end): - line_list = [] - log_path = self.get_log_file_path() - if begin and end: - cmd = f"cat {log_path} | tail -n +{begin}| head -n {end-begin+1}" - elif begin: - cmd = f"cat {log_path} | tail -n +{begin}" - elif end: - cmd = f"cat {log_path} | head -n {end}" - else: - cmd = f"cat {log_path}" - lines = self.execute(cmd) - if lines: - line_list = [] - line_num = begin if begin else 1 - for line in lines.split("\n"): - line = replace_ip(line) - line_list.append({"line_num": line_num, "content": line}) - line_num += 1 - return line_list - - def get_size(self): - try: - return int(self.execute(f"cat {self.get_log_file_path()} | wc -l").strip()) - except: - return 0 - - @staticmethod - def execute(cmd): - res = subprocess.run( - cmd, shell=True, universal_newlines=True, - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, - ) - return res.stdout diff --git a/python/fate_flow/utils/log_utils.py b/python/fate_flow/utils/log_utils.py index 26b5070bf..01991524e 100644 --- a/python/fate_flow/utils/log_utils.py +++ b/python/fate_flow/utils/log_utils.py @@ -19,8 +19,8 @@ import traceback import logging -from fate_arch.common.log import LoggerFactory, getLogger -from fate_flow.utils.base_utils import get_fate_flow_directory +from fate_flow.runtime.system_settings import FATE_FLOW_LOG_DIR, LOG_DIR +from fate_flow.utils.log import LoggerFactory, getLogger def ready_log(msg, job=None, task=None, role=None, party_id=None, detail=None): @@ -67,14 +67,9 @@ def exception_to_trace_string(ex): return "".join(traceback.TracebackException.from_exception(ex).format()) -def get_logger_base_dir(): - job_log_dir = get_fate_flow_directory('logs') - return job_log_dir - - def get_job_logger(job_id, log_type): - fate_flow_log_dir = get_fate_flow_directory('logs', 'fate_flow') - job_log_dir = get_fate_flow_directory('logs', job_id) + fate_flow_log_dir = FATE_FLOW_LOG_DIR + job_log_dir = os.path.join(LOG_DIR, job_id) if not job_id: log_dirs = [fate_flow_log_dir] else: @@ -102,9 +97,9 @@ def get_job_logger(job_id, log_type): return logger -def schedule_logger(job_id=None, delete=False): +def schedule_logger(job_id=None, delete=False, name="fate_flow_schedule"): if not job_id: - return getLogger("fate_flow_schedule") + return getLogger(name) else: if delete: with LoggerFactory.lock: diff --git a/python/fate_flow/utils/model_utils.py b/python/fate_flow/utils/model_utils.py deleted file mode 100644 index d746eae0e..000000000 --- a/python/fate_flow/utils/model_utils.py +++ /dev/null @@ -1,282 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import glob - -from fate_arch.common.base_utils import json_loads - -from fate_flow.db.db_models import DB, MachineLearningModelInfo as MLModel -from fate_flow.model.sync_model import SyncModel -from fate_flow.pipelined_model.pipelined_model import PipelinedModel -from fate_flow.scheduler.cluster_scheduler import ClusterScheduler -from fate_flow.settings import ENABLE_MODEL_STORE, stat_logger -from fate_flow.utils import schedule_utils -from fate_flow.utils.base_utils import compare_version, get_fate_flow_directory - - -def all_party_key(all_party): - """ - Join all party as party key - :param all_party: - "role": { - "guest": [9999], - "host": [10000], - "arbiter": [10000] - } - :return: - """ - if not all_party: - all_party_key = 'all' - elif isinstance(all_party, dict): - sorted_role_name = sorted(all_party.keys()) - all_party_key = '#'.join([ - ('%s-%s' % ( - role_name, - '_'.join([str(p) for p in sorted(set(all_party[role_name]))])) - ) - for role_name in sorted_role_name]) - else: - all_party_key = None - return all_party_key - - -def gen_party_model_id(model_id, role, party_id): - return '#'.join([role, str(party_id), model_id]) if model_id else None - - -def gen_model_id(all_party): - return '#'.join([all_party_key(all_party), "model"]) - - -@DB.connection_context() -def query_model_info_from_db(query_filters=None, **kwargs): - conditions = [] - filters = [] - - for k, v in kwargs.items(): - k = f'f_{k}' - if hasattr(MLModel, k): - conditions.append(getattr(MLModel, k) == v) - - for k in query_filters: - k = f'f_{k}' - if hasattr(MLModel, k): - filters.append(getattr(MLModel, k)) - - models = MLModel.select(*filters) - if conditions: - models = models.where(*conditions) - models = [model.to_dict() for model in models] - - if not models: - return 100, 'Query model info failed, cannot find model from db.', [] - return 0, 'Query model info from db success.', models - - -def query_model_info_from_file(model_id='*', model_version='*', role='*', party_id='*', query_filters=None, save_to_db=False, **kwargs): - fp_list = glob.glob(f"{get_fate_flow_directory('model_local_cache')}/{role}#{party_id}#{model_id}/{model_version}") - - models = [] - for fp in fp_list: - _, party_model_id, model_version = fp.rsplit('/', 2) - role, party_id, model_id = party_model_id.split('#', 2) - - pipeline_model = PipelinedModel(model_id=party_model_id, model_version=model_version) - if not pipeline_model.exists(): - continue - - model_info = gather_model_info_data(pipeline_model) - - if save_to_db: - try: - save_model_info(model_info) - except Exception as e: - stat_logger.exception(e) - - if query_filters: - model_info = {k: v for k, v in model_info.items() if k in query_filters} - - models.append(model_info) - - if not models: - return 100, 'Query model info failed, cannot find model from local model files.', [] - return 0, 'Query model info from local model success.', models - - -def gather_model_info_data(model: PipelinedModel): - pipeline = model.read_pipeline_model() - - model_info = {} - for attr, field in pipeline.ListFields(): - if isinstance(field, bytes): - field = json_loads(field) - model_info[f'f_{attr.name}'] = field - - model_info['f_job_id'] = model_info['f_model_version'] - model_info['f_role'] = model.role - model_info['f_party_id'] = model.party_id - # backward compatibility - model_info['f_runtime_conf'] = model_info['f_train_runtime_conf'] - model_info['f_size'] = model.calculate_model_file_size() - - if compare_version(model_info['f_fate_version'], '1.5.1') == 'lt': - model_info['f_roles'] = model_info.get('f_train_runtime_conf', {}).get('role', {}) - model_info['f_initiator_role'] = model_info.get('f_train_runtime_conf', {}).get('initiator', {}).get('role') - model_info['f_initiator_party_id'] = model_info.get('f_train_runtime_conf', {}).get('initiator', {}).get('party_id') - - return model_info - - -def query_model_detail(model_id, model_version, **kwargs): - model_detail = {} - retcode, retmsg, model_infos = query_model_info(model_id=model_id, model_version=model_version) - if not model_infos: - return retcode, retmsg, model_detail - model_info = model_infos[0] - model_detail["runtime_conf"] = model_info.get("f_runtime_conf") or model_info.get("f_train_runtime_conf") - model_detail["dsl"] = model_info.get("f_train_dsl") - model_detail["inference_dsl"] = model_info.get("f_inference_dsl", {}) - is_parent = model_info.get("f_parent") - model_detail["component_info"] = get_component_list(model_detail["runtime_conf"], model_detail["dsl"], is_parent) - model_detail["inference_component_info"] = get_component_list(model_detail["runtime_conf"], model_detail["inference_dsl"], is_parent) - - return retcode, retmsg, model_detail - - -def get_component_list(conf, dsl, is_train): - job_type = "train" - if not is_train: - job_type = "predict" - dsl_parser = schedule_utils.get_job_dsl_parser(dsl=dsl, - runtime_conf=conf, - train_runtime_conf=conf, - job_type=job_type - ) - name_component_maps, hierarchical_structure = dsl_parser.get_dsl_hierarchical_structure() - return [{"component_name": k, "module": v["module"], "index": get_component_index(k, hierarchical_structure)} - for k, v in dsl.get("components", {}).items()] - - -def get_component_index(component_name, hierarchical_structure): - for index, cpn_list in enumerate(hierarchical_structure): - if component_name in cpn_list: - return index - - -def query_model_info(**kwargs): - file_only = kwargs.pop('file_only', False) - kwargs['query_filters'] = set(kwargs['query_filters']) if kwargs.get('query_filters') else set() - - if not file_only: - retcode, retmsg, data = query_model_info_from_db(**kwargs) - if not retcode: - return retcode, retmsg, data - - kwargs['save_to_db'] = True - - retcode, retmsg, data = query_model_info_from_file(**kwargs) - if not retcode: - return retcode, retmsg, data - - return 100, ( - 'Query model info failed, cannot find model from db and local model files. ' - 'Try use both model id and model version to query model info from local models.' - ), [] - - -def save_model_info(model_info): - model_info = {k if k.startswith('f_') else f'f_{k}': v for k, v in model_info.items()} - - with DB.connection_context(): - MLModel.insert(**model_info).on_conflict(preserve=( - 'f_update_time', - 'f_update_date', - *model_info.keys(), - )).execute() - - if ENABLE_MODEL_STORE: - sync_model = SyncModel( - role=model_info['f_role'], party_id=model_info['f_party_id'], - model_id=model_info['f_model_id'], model_version=model_info['f_model_version'], - ) - sync_model.upload(True) - - ClusterScheduler.cluster_command('/model/service/register', { - 'party_model_id': gen_party_model_id( - model_info['f_model_id'], - model_info['f_role'], - model_info['f_party_id'], - ), - 'model_version': model_info['f_model_version'], - }) - - -def check_if_parent_model(pipeline): - if compare_version(pipeline.fate_version, '1.5.0') == 'gt': - if pipeline.parent: - return True - return False - - -def check_before_deploy(pipeline_model: PipelinedModel): - pipeline = pipeline_model.read_pipeline_model() - - if compare_version(pipeline.fate_version, '1.5.0') == 'gt': - if pipeline.parent: - return True - elif compare_version(pipeline.fate_version, '1.5.0') == 'eq': - return True - return False - - -def check_if_deployed(role, party_id, model_id, model_version): - party_model_id = gen_party_model_id(model_id=model_id, role=role, party_id=party_id) - pipeline_model = PipelinedModel(model_id=party_model_id, model_version=model_version) - if not pipeline_model.exists(): - raise FileNotFoundError(f"Model {party_model_id} {model_version} not exists in model local cache.") - - pipeline = pipeline_model.read_pipeline_model() - if compare_version(pipeline.fate_version, '1.5.0') == 'gt': - train_runtime_conf = json_loads(pipeline.train_runtime_conf) - if str(train_runtime_conf.get('dsl_version', '1')) != '1': - if pipeline.parent: - return False - return True - - -@DB.connection_context() -def models_group_by_party_model_id_and_model_version(): - args = [ - MLModel.f_role, - MLModel.f_party_id, - MLModel.f_model_id, - MLModel.f_model_version, - ] - return MLModel.select(*args).group_by(*args) - - -@DB.connection_context() -def get_job_configuration_from_model(job_id, role, party_id): - retcode, retmsg, data = query_model_info( - model_version=job_id, role=role, party_id=party_id, - query_filters=['train_dsl', 'dsl', 'train_runtime_conf', 'runtime_conf'], - ) - if not data: - return {}, {}, {} - - dsl = data[0].get('train_dsl') if data[0].get('train_dsl') else data[0].get('dsl') - runtime_conf = data[0].get('runtime_conf') - train_runtime_conf = data[0].get('train_runtime_conf') - return dsl, runtime_conf, train_runtime_conf diff --git a/python/fate_flow/utils/password_utils.py b/python/fate_flow/utils/password_utils.py new file mode 100644 index 000000000..6db4ff75e --- /dev/null +++ b/python/fate_flow/utils/password_utils.py @@ -0,0 +1,43 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os.path +from importlib import import_module + +from fate_flow.runtime.system_settings import ENCRYPT_CONF +from fate_flow.utils.conf_utils import conf_realpath + + +def decrypt_database_config(database, passwd_key="passwd", decrypt_key=""): + database[passwd_key] = decrypt_password(database[passwd_key], key=decrypt_key) + return database + + +def decrypt_password(password, key=""): + if not ENCRYPT_CONF or not key or key not in ENCRYPT_CONF: + return password + encrypt_module = ENCRYPT_CONF.get(key).get("module", "") + private_path = ENCRYPT_CONF.get(key).get("private_path", "") + if not encrypt_module: + raise ValueError(f"module is {encrypt_module}") + if not private_path: + raise ValueError(f"private_path is {private_path}") + if not os.path.isabs(private_path): + private_path = conf_realpath(private_path) + with open(private_path) as f: + private_key = f.read() + module_func = encrypt_module.split("#") + encrypt_func = getattr(import_module(module_func[0]), module_func[1]) + return encrypt_func(private_key, password) diff --git a/python/fate_flow/utils/permission_utils.py b/python/fate_flow/utils/permission_utils.py index d6ff55c53..fc28bce4f 100644 --- a/python/fate_flow/utils/permission_utils.py +++ b/python/fate_flow/utils/permission_utils.py @@ -1,39 +1,63 @@ -from fate_flow.db.component_registry import ComponentRegistry -from fate_flow.entity.permission_parameters import DataSet -from fate_flow.hook.common.parameters import PermissionCheckParameters -from fate_flow.utils import schedule_utils, job_utils - +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from functools import wraps +from flask import request as flask_request -def get_permission_parameters(role, party_id, src_role, src_party_id, job_info) -> PermissionCheckParameters: - dsl = job_info['dsl'] - runtime_conf = job_info['runtime_conf'] - train_runtime_conf = job_info['train_runtime_conf'] +from fate_flow.controller.parser import JobParser +from fate_flow.entity.code import ReturnCode +from fate_flow.entity.spec.dag import DAGSchema +from fate_flow.hook import HookManager +from fate_flow.hook.common.parameters import PermissionCheckParameters +from fate_flow.manager.service.provider_manager import ProviderManager +from fate_flow.runtime.system_settings import PERMISSION_SWITCH +from fate_flow.utils.api_utils import API - dsl_parser = schedule_utils.get_job_dsl_parser( - dsl=dsl, - runtime_conf=runtime_conf, - train_runtime_conf=train_runtime_conf - ) - provider_detail = ComponentRegistry.REGISTRY - job_providers = dsl_parser.get_job_providers(provider_detail=provider_detail) - component_parameters = job_utils.get_component_parameters(job_providers, dsl_parser, provider_detail, role, int(party_id)) - dataset_dict = job_utils.get_job_dataset(False, role, int(party_id), runtime_conf.get("role"), dsl_parser.get_args_input()) - dataset_list = [] - if dataset_dict.get(role, {}).get(int(party_id)): - for _, v in dataset_dict[role][int(party_id)].items(): - dataset_list.append(DataSet(namespace=v.split('.')[0], name=v.split('.')[1])) - component_list = job_utils.get_job_all_components(dsl) +def get_permission_parameters(role, party_id, initiator_party_id, job_info) -> PermissionCheckParameters: + dag_schema = DAGSchema(**job_info) + job_parser = JobParser(dag_schema) + component_list = job_parser.component_ref_list(role, party_id) + fate_component_list = set(component_list) - set(ProviderManager.get_flow_components()) + dataset_list = job_parser.dataset_list(role, party_id) + component_parameters = job_parser.role_parameters(role, party_id) return PermissionCheckParameters( - src_role=src_role, - src_party_id=src_party_id, - role=role, - party_id=party_id, - initiator=runtime_conf['initiator'], - roles=runtime_conf['role'], - component_list=component_list, + initiator_party_id=initiator_party_id, + roles=dag_schema.dag.parties, + component_list=fate_component_list, dataset_list=dataset_list, - runtime_conf=runtime_conf, - dsl=dsl, + dag_schema=dag_schema.dict(), component_parameters=component_parameters ) + + +def create_job_request_check(func): + @wraps(func) + def _wrapper(*_args, **_kwargs): + party_id = _kwargs.get("party_id") + role = _kwargs.get("role") + body = flask_request.json + headers = flask_request.headers + initiator_party_id = headers.get("initiator_party_id") + + # permission check + if PERMISSION_SWITCH: + permission_return = HookManager.permission_check(get_permission_parameters( + role, party_id, initiator_party_id, body + )) + if permission_return.code != ReturnCode.Base.SUCCESS: + return API.Output.fate_flow_exception(permission_return) + return func(*_args, **_kwargs) + return _wrapper diff --git a/python/fate_flow/utils/process_utils.py b/python/fate_flow/utils/process_utils.py index 6d6d0b546..df736797a 100644 --- a/python/fate_flow/utils/process_utils.py +++ b/python/fate_flow/utils/process_utils.py @@ -14,31 +14,40 @@ # limitations under the License. # import errno +import json import os import subprocess +import time + import psutil -from fate_flow.db.job_default_config import JobDefaultConfig +from fate_flow.entity.code import KillProcessRetCode +from fate_flow.utils.log import getLogger from fate_flow.utils.log_utils import schedule_logger from fate_flow.db.db_models import Task -from fate_flow.entity.types import KillProcessRetCode, ProcessRole -from fate_flow.settings import SUBPROCESS_STD_LOG_NAME -from fate_flow.settings import stat_logger +from fate_flow.entity.types import ProcessRole + +stat_logger = getLogger() -def run_subprocess(job_id, config_dir, process_cmd, added_env: dict = None, log_dir=None, cwd_dir=None, process_name="", process_id=""): +def run_subprocess( + job_id, config_dir, process_cmd, process_name, added_env: dict = None, std_dir=None, cwd_dir=None, stderr=None +): logger = schedule_logger(job_id) if job_id else stat_logger process_cmd = [str(cmd) for cmd in process_cmd] logger.info("start process command: \n{}".format(" ".join(process_cmd))) os.makedirs(config_dir, exist_ok=True) - if not log_dir: - log_dir = config_dir - if log_dir: - os.makedirs(log_dir, exist_ok=True) - std_path = get_std_path(log_dir=log_dir, process_name=process_name, process_id=process_id) + os.makedirs(std_dir, exist_ok=True) + if not std_dir: + std_dir = config_dir + std_path = get_std_path(std_dir=std_dir, process_name=process_name) + std = open(std_path, 'w') - pid_path = os.path.join(config_dir, f"{process_name}_pid") + if not stderr: + stderr = std + pid_path = os.path.join(config_dir, "pid", f"{process_name}") + os.makedirs(os.path.dirname(pid_path), exist_ok=True) if os.name == 'nt': startupinfo = subprocess.STARTUPINFO() @@ -51,16 +60,15 @@ def run_subprocess(job_id, config_dir, process_cmd, added_env: dict = None, log_ subprocess_env["PROCESS_ROLE"] = ProcessRole.WORKER.value if added_env: for name, value in added_env.items(): + if not value: + continue if name.endswith("PATH") and subprocess_env.get(name) is not None: value += ':' + subprocess_env[name] subprocess_env[name] = value - - if not JobDefaultConfig.task_process_classpath: - subprocess_env.pop("CLASSPATH", None) - + logger.info(f"RUN ENV：{json.dumps(subprocess_env)}") p = subprocess.Popen(process_cmd, stdout=std, - stderr=std, + stderr=stderr, startupinfo=startupinfo, cwd=cwd_dir, env=subprocess_env @@ -95,10 +103,17 @@ def check_process(pid, task: Task = None, expected_cmdline: list = None): ret = True if ret and task is not None: p = get_process_instance(pid) - return is_task_executor_process(task=task, process=p) + if p: + return True + else: + return False elif ret and expected_cmdline is not None: p = get_process_instance(pid) - return check_process_by_cmdline(actual=p.cmdline(), expected=expected_cmdline) + try: + return check_process_by_cmdline(actual=p.cmdline(), expected=expected_cmdline) + except psutil.NoSuchProcess: + stat_logger.warning(f"no such process {pid}") + return False else: return ret @@ -121,15 +136,8 @@ def check_process_by_cmdline(actual: list, expected: list): return True -def get_std_path(log_dir, process_name="", process_id=""): - std_log_path = f"{process_name}_{process_id}_{SUBPROCESS_STD_LOG_NAME}" if process_name else SUBPROCESS_STD_LOG_NAME - return os.path.join(log_dir, std_log_path) - - -def get_subprocess_std(log_dir, process_name="", process_id=""): - with open(get_std_path(log_dir, process_name, process_id), "r") as fr: - text = fr.read() - return text +def get_std_path(std_dir, process_name): + return os.path.join(std_dir, process_name) def wait_child_process(signum, frame): @@ -159,7 +167,6 @@ def is_task_executor_process(task: Task, process: psutil.Process): cmdline = process.cmdline() except Exception as e: # Not sure whether the process is a task executor process, operations processing is required - schedule_logger(task.f_job_id).warning(e) return False else: schedule_logger(task.f_job_id).info(cmdline) @@ -210,6 +217,34 @@ def kill_task_executor_process(task: Task, only_child=False): raise e +def kill(p, wait_before_terminate=10, wait_before_kill=10): + # wait and check + for _ in range(wait_before_terminate): + if p.is_running(): + time.sleep(1) + else: + break + try: + # send sigterm, gracefully stop + if p.is_running(): + p.terminate() + except: + pass + finally: + # gracefully stop may takes few seconds, wati and check again + for _ in range(wait_before_kill): + if p.is_running(): + time.sleep(1) + else: + break + try: + # nothing could do now, kill anyway + if p.is_running(): + p.kill() + except: + pass + + def kill_process(process: psutil.Process = None, pid: int = None, expected_cmdline: list = None): process = process if process is not None else get_process_instance(pid) if process is None: diff --git a/python/fate_flow/utils/requests_utils.py b/python/fate_flow/utils/requests_utils.py index 5e1f47228..78b0a292e 100644 --- a/python/fate_flow/utils/requests_utils.py +++ b/python/fate_flow/utils/requests_utils.py @@ -15,17 +15,9 @@ # import functools import json -from base64 import b64encode -from hmac import HMAC -from time import time -from urllib.parse import quote, urlencode -from uuid import uuid1 - import requests -from fate_arch.common.base_utils import CustomJSONEncoder -from fate_flow.settings import CLIENT_AUTHENTICATION, HTTP_APP_KEY, HTTP_SECRET_KEY - +from fate_flow.utils.base_utils import CustomJSONEncoder requests.models.complexjson.dumps = functools.partial(json.dumps, cls=CustomJSONEncoder) @@ -34,27 +26,7 @@ def request(**kwargs): sess = requests.Session() stream = kwargs.pop('stream', sess.stream) timeout = kwargs.pop('timeout', None) - kwargs['headers'] = {k.replace('_', '-').upper(): v for k, v in kwargs.get('headers', {}).items()} + if kwargs.get('headers'): + kwargs['headers'] = {k.replace('_', '-').upper(): v for k, v in kwargs.get('headers', {}).items()} prepped = requests.Request(**kwargs).prepare() - - if CLIENT_AUTHENTICATION and HTTP_APP_KEY and HTTP_SECRET_KEY: - timestamp = str(round(time() * 1000)) - nonce = str(uuid1()) - signature = b64encode(HMAC(HTTP_SECRET_KEY.encode('ascii'), b'\n'.join([ - timestamp.encode('ascii'), - nonce.encode('ascii'), - HTTP_APP_KEY.encode('ascii'), - prepped.path_url.encode('ascii'), - prepped.body if kwargs.get('json') else b'', - urlencode(sorted(kwargs['data'].items()), quote_via=quote, safe='-._~').encode('ascii') - if kwargs.get('data') and isinstance(kwargs['data'], dict) else b'', - ]), 'sha1').digest()).decode('ascii') - - prepped.headers.update({ - 'TIMESTAMP': timestamp, - 'NONCE': nonce, - 'APP-KEY': HTTP_APP_KEY, - 'SIGNATURE': signature, - }) - return sess.send(prepped, stream=stream, timeout=timeout) diff --git a/python/fate_flow/utils/runtime_conf_parse_util.py b/python/fate_flow/utils/runtime_conf_parse_util.py deleted file mode 100644 index fb4bf2b1b..000000000 --- a/python/fate_flow/utils/runtime_conf_parse_util.py +++ /dev/null @@ -1,599 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import copy -from fate_arch.abc import Components -from fate_flow.component_env_utils import provider_utils -from fate_flow.entity import ComponentProvider -from fate_flow.db.component_registry import ComponentRegistry - - -class RuntimeConfParserUtil(object): - @classmethod - def get_input_parameters(cls, submit_dict, components=None): - return RuntimeConfParserV2.get_input_parameters(submit_dict, components=components) - - @classmethod - def get_job_parameters(cls, submit_dict, conf_version=1): - if conf_version == 1: - return RuntimeConfParserV1.get_job_parameters(submit_dict) - else: - return RuntimeConfParserV2.get_job_parameters(submit_dict) - - @staticmethod - def merge_dict(dict1, dict2): - merge_ret = {} - key_set = dict1.keys() | dict2.keys() - for key in key_set: - if key in dict1 and key in dict2: - val1 = dict1.get(key) - val2 = dict2.get(key) - if isinstance(val1, dict): - merge_ret[key] = RuntimeConfParserUtil.merge_dict(val1, val2) - else: - merge_ret[key] = val2 - elif key in dict1: - merge_ret[key] = dict1.get(key) - else: - merge_ret[key] = dict2.get(key) - - return merge_ret - - @staticmethod - def generate_predict_conf_template(predict_dsl, train_conf, model_id, model_version): - return RuntimeConfParserV2.generate_predict_conf_template(predict_dsl, train_conf, model_id, model_version) - - @staticmethod - def get_module_name(module, role, provider: Components): - return provider.get(module, ComponentRegistry.get_provider_components(provider.provider_name, provider.provider_version)).get_run_obj_name(role) - - @staticmethod - def get_component_parameters( - provider, - runtime_conf, - module, - alias, - redundant_param_check, - local_role, - local_party_id, - parse_user_specified_only, - pre_parameters=None - ): - provider_components = ComponentRegistry.get_provider_components( - provider.provider_name, provider.provider_version - ) - support_roles = provider.get(module, provider_components).get_supported_roles() - if runtime_conf["role"] is not None: - support_roles = [r for r in runtime_conf["role"] if r in support_roles] - role_on_module = copy.deepcopy(runtime_conf["role"]) - for role in runtime_conf["role"]: - if role not in support_roles: - del role_on_module[role] - - if local_role not in role_on_module: - return {} - - conf = dict() - for key, value in runtime_conf.items(): - if key not in [ - "algorithm_parameters", - "role_parameters", - "component_parameters", - ]: - conf[key] = value - - conf["role"] = role_on_module - conf["local"] = runtime_conf.get("local", {}) - conf["local"].update({"role": local_role, "party_id": local_party_id}) - conf["module"] = module - conf["CodePath"] = provider.get(module, provider_components).get_run_obj_name( - local_role - ) - - param_class = provider.get(module, provider_components).get_param_obj(alias) - role_idx = role_on_module[local_role].index(local_party_id) - - user_specified_parameters = dict() - - if pre_parameters: - if parse_user_specified_only: - user_specified_parameters.update( - pre_parameters.get("ComponentParam", {}) - ) - else: - param_class = param_class.update( - pre_parameters.get("ComponentParam", {}) - ) - - common_parameters = ( - runtime_conf.get("component_parameters", {}).get("common", {}).get(alias, {}) - ) - - if parse_user_specified_only: - user_specified_parameters.update(common_parameters) - else: - param_class = param_class.update( - common_parameters, not redundant_param_check - ) - - # update role parameters - for role_id, role_id_parameters in ( - runtime_conf.get("component_parameters", {}) - .get("role", {}) - .get(local_role, {}) - .items() - ): - if role_id == "all" or str(role_idx) in role_id.split("|"): - parameters = role_id_parameters.get(alias, {}) - if parse_user_specified_only: - user_specified_parameters.update(parameters) - else: - param_class.update(parameters, not redundant_param_check) - - if not parse_user_specified_only: - conf["ComponentParam"] = param_class.as_dict() - param_class.check() - else: - conf["ComponentParam"] = user_specified_parameters - - return conf - - @staticmethod - def convert_parameters_v1_to_v2(party_idx, parameter_v1, not_builtin_vars): - parameter_v2 = {} - for key, values in parameter_v1.items(): - # stop here, values support to be a list - if key not in not_builtin_vars: - parameter_v2[key] = values[party_idx] - else: - parameter_v2[key] = RuntimeConfParserUtil.convert_parameters_v1_to_v2(party_idx, values, not_builtin_vars) - - return parameter_v2 - - @staticmethod - def get_v1_role_parameters(provider, component, runtime_conf, dsl): - component_role_parameters = dict() - if "role_parameters" not in runtime_conf: - return component_role_parameters - - role_parameters = runtime_conf["role_parameters"] - module = dsl["components"][component]["module"] - if module == "Reader": - data_key = dsl["components"][component]["output"]["data"][0] - - for role, role_params in role_parameters.items(): - if not role_params.get("args", {}).get("data", {}).get(data_key): - continue - - component_role_parameters[role] = dict() - dataset = role_params["args"]["data"][data_key] - for idx, table in enumerate(dataset): - component_role_parameters[role][str(idx)] = {component: {"table": table}} - else: - provider_components = ComponentRegistry.get_provider_components( - provider.provider_name, provider.provider_version - ) - param_class = provider.get(module, provider_components).get_param_obj(component) - extract_not_builtin = getattr(param_class, "extract_not_builtin", None) - not_builtin_vars = extract_not_builtin() if extract_not_builtin is not None else {} - - for role, role_params in role_parameters.items(): - params = role_params.get(component, {}) - if not params: - continue - - component_role_parameters[role] = dict() - party_num = len(runtime_conf["role"][role]) - - for party_idx in range(party_num): - party_param = RuntimeConfParserUtil.convert_parameters_v1_to_v2(party_idx, params, not_builtin_vars) - component_role_parameters[role][str(party_idx)] = {component: party_param} - - return component_role_parameters - - @staticmethod - def get_job_providers_by_dsl(dsl, provider_detail): - provider_info = {} - global_provider_name = None - global_provider_version = None - if "provider" in dsl: - global_provider_msg = dsl["provider"].split("@", -1) - if global_provider_msg[0] == "@" or len(global_provider_msg) > 2: - raise ValueError("Provider format should be provider_name@provider_version or provider_name, " - "@provider_version is not supported") - if len(global_provider_msg) == 1: - global_provider_name = global_provider_msg[0] - else: - global_provider_name, global_provider_version = global_provider_msg - - for component in dsl["components"]: - module = dsl["components"][component]["module"] - provider_config = dsl["components"][component].get("provider") - name, version = RuntimeConfParserUtil.get_component_provider_by_user_conf(component, - module, - provider_config, - provider_detail, - global_provider_name, - global_provider_version) - - provider_info.update({component: { - "module": module, - "provider": { - "name": name, - "version": version - } - }}) - - return provider_info - - @classmethod - def get_job_providers(cls, dsl, provider_detail, submit_dict=None, local_role=None, local_party_id=None): - provider_info = cls.get_job_providers_by_dsl(dsl, provider_detail) - if submit_dict is None: - return provider_info - else: - if local_party_id is None or local_role is None \ - or local_role not in submit_dict["role"] or \ - (str(local_party_id) not in submit_dict["role"][local_role] - and int(local_party_id) not in submit_dict["role"][local_role]): - raise ValueError("when parse provider from conf, local role & party_id should should be None") - - provider_info_all_party = {} - dsl_version = submit_dict.get("dsl_version", 1) - if dsl_version == 1 or "provider" not in submit_dict: - for role in submit_dict["role"]: - party_id_list = submit_dict["role"][role] - provider_info_all_party[role] = {party_id: dict() for party_id in party_id_list} - - provider_info_all_party[local_role][local_party_id] = provider_info - else: - provider_config = submit_dict["provider"] - common_provider_config = provider_config.get("common", {}) - other_party_provider_config = dict() - if common_provider_config: - for component, provider_msg in common_provider_config.items(): - if component not in provider_info: - raise ValueError(f"Redundant omponent {component} is not found in dsl") - - module = provider_info[component]["module"] - name, version = cls.get_component_provider_by_user_conf(component, - module, - provider_msg, - provider_detail) - - provider_info[component]["provider"] = dict(name=name, version=version) - - other_name, other_version = cls.get_component_provider_by_user_conf(component, - module, - provider_msg) - other_party_provider_config[component] = { - "module": module, - "provider": { - "name": other_name, - "version": other_version - } - } - - provider_info_all_party[local_role]= {local_party_id : copy.deepcopy(provider_info)} - - for role in submit_dict["role"]: - if role not in provider_info_all_party: - provider_info_all_party[role] = {} - role_provider_config = provider_config.get("role", {}).get(role, {}) - for idx, party_id in enumerate(submit_dict["role"][role]): - if role == local_role and party_id == local_party_id: - provider_info_party = copy.deepcopy(provider_info) - else: - provider_info_party = copy.deepcopy(other_party_provider_config) - - for role_id, role_id_provider_config in role_provider_config.items(): - if role_id == "all" or str(idx) in role_id.split("|", -1): - for component, provider_msg in role_id_provider_config.items(): - module = dsl["components"][component]["module"] - detail_info = provider_detail if role == role and party_id == local_party_id else None - name, version = cls.get_component_provider_by_user_conf(component, - module, - provider_msg, - provider_detail=detail_info) - if component not in provider_info_party: - provider_info_party[component] = dict(module=module) - - provider_info_party[component]["provider"] = dict(name=name, version=version) - - provider_info_all_party[role][party_id] = provider_info_party - - return provider_info_all_party - - @staticmethod - def get_component_provider_by_user_conf(component, module, provider_config, provider_detail=None, - default_name=None, default_version=None): - name, version = None, None - if provider_config: - provider_msg = provider_config.split("@", -1) - if provider_config[0] == "@" or len(provider_msg) > 2: - raise ValueError("Provider format should be provider_name@provider_version or provider_name, " - "@provider_version is not supported") - if len(provider_msg) == 2: - name, version = provider_config.split("@", -1) - else: - name = provider_msg[0] - - if not name: - if default_name: - name = default_name - version = default_version - - if provider_detail is None: - return name, version - - if name and name not in provider_detail["components"][module]["support_provider"]: - raise ValueError(f"Provider: {name} does not support in {module}, please register") - if version and version not in provider_detail["providers"][name]: - raise ValueError(f"Provider: {name} version: {version} does not support in {module}, please register") - - if name and not version: - version = RuntimeConfParserUtil.get_component_provider(alias=component, - module=module, - provider_detail=provider_detail, - name=name) - elif not name and not version: - name, version = RuntimeConfParserUtil.get_component_provider(alias=component, - module=module, - provider_detail=provider_detail) - - return name, version - - @staticmethod - def get_component_provider(alias, module, provider_detail, detect=True, name=None): - if module not in provider_detail["components"]: - if detect: - raise ValueError(f"component {alias}, module {module}'s provider does not exist") - else: - return None - - if name is None: - name = provider_detail["components"][module]["default_provider"] - version = provider_detail["providers"][name]["default"]["version"] - return name, version - else: - if name not in provider_detail["components"][module]["support_provider"]: - raise ValueError(f"Provider {name} does not support, please register in fate-flow") - version = provider_detail["providers"][name]["default"]["version"] - - return version - - @staticmethod - def instantiate_component_provider(provider_detail, alias=None, module=None, provider_name=None, - provider_version=None, local_role=None, local_party_id=None, - detect=True, provider_cache=None, job_parameters=None): - if provider_name and provider_version: - provider_path = provider_detail["providers"][provider_name][provider_version]["path"] - provider = provider_utils.get_provider_interface(ComponentProvider(name=provider_name, - version=provider_version, - path=provider_path, - class_path=ComponentRegistry.get_default_class_path())) - if provider_cache is not None: - if provider_name not in provider_cache: - provider_cache[provider_name] = {} - - provider_cache[provider_name][provider_version] = provider - - return provider - - provider_name, provider_version = RuntimeConfParserUtil.get_component_provider(alias=alias, - module=module, - provider_detail=provider_detail, - detect=detect) - - return RuntimeConfParserUtil.instantiate_component_provider(provider_detail, - provider_name=provider_name, - provider_version=provider_version) - - @classmethod - def merge_predict_runtime_conf(cls, train_conf, predict_conf): - runtime_conf = copy.deepcopy(train_conf) - train_role = train_conf.get("role") - predict_role = predict_conf.get("role") - if len(train_conf) < len(predict_role): - raise ValueError(f"Predict roles is {predict_role}, train roles is {train_conf}, " - "predict roles should be subset of train role") - - for role in train_role: - if role not in predict_role: - del runtime_conf["role"][role] - - if runtime_conf.get("job_parameters", {}).get("role", {}).get(role): - del runtime_conf["job_parameters"]["role"][role] - - if runtime_conf.get("component_parameters", {}).get("role", {}).get(role): - del runtime_conf["component_parameters"]["role"][role] - - continue - - train_party_ids = train_role[role] - predict_party_ids = predict_role[role] - - diff = False - for idx, party_id in enumerate(predict_party_ids): - if party_id not in train_party_ids: - raise ValueError(f"Predict role: {role} party_id: {party_id} not occurs in training") - if train_party_ids[idx] != party_id: - diff = True - - if not diff and len(train_party_ids) == len(predict_party_ids): - continue - - for p_type in ["job_parameters", "component_parameters"]: - if not runtime_conf.get(p_type, {}).get("role", {}).get(role): - continue - - conf = runtime_conf[p_type]["role"][role] - party_keys = conf.keys() - new_conf = {} - for party_key in party_keys: - party_list = party_key.split("|", -1) - new_party_list = [] - for party in party_list: - party_id = train_party_ids[int(party)] - if party_id in predict_party_ids: - new_idx = predict_party_ids.index(party_id) - new_party_list.append(str(new_idx)) - - if not new_party_list: - continue - - new_party_key = new_party_list[0] if len(new_party_list) == 1 else "|".join(new_party_list) - - if new_party_key not in new_conf: - new_conf[new_party_key] = {} - new_conf[new_party_key].update(conf[party_key]) - - runtime_conf[p_type]["role"][role] = new_conf - - runtime_conf = cls.merge_dict(runtime_conf, predict_conf) - - return runtime_conf - - @staticmethod - def get_model_loader_alias(component_name, runtime_conf, local_role, local_party_id): - role_params = runtime_conf.get("component_parameters", {}).get("role", {}).get("local_role") - if not role_params: - return runtime_conf.get("component_parameters", {}).\ - get("common", {}).get(component_name, {}).get("component_name") - - party_idx = runtime_conf.get("role").get(local_role).index(local_party_id) - for id_list, params in role_params.times(): - ids = id_list.split("|", -1) - if ids == "all" or str(party_idx) in ids: - if params.get(component_name, {}).get("component_name"): - model_load_alias = params.get(component_name, {}).get("component_name") - return model_load_alias - - return runtime_conf.get("component_parameters", {}). \ - get("common", {}).get(component_name, {}).get("component_name") - -class RuntimeConfParserV1(object): - @staticmethod - def get_job_parameters(submit_dict): - ret = {} - job_parameters = submit_dict.get("job_parameters", {}) - for role in submit_dict["role"]: - party_id_list = submit_dict["role"][role] - ret[role] = {party_id: copy.deepcopy(job_parameters) for party_id in party_id_list} - - return ret - - -class RuntimeConfParserV2(object): - @classmethod - def get_input_parameters(cls, submit_dict, components=None): - if submit_dict.get("component_parameters", {}).get("role") is None or components is None: - return {} - - roles = submit_dict["component_parameters"]["role"].keys() - if not roles: - return {} - - input_parameters = {"dsl_version": 2} - - cpn_dict = {} - for reader_cpn in components: - cpn_dict[reader_cpn] = {} - - for role in roles: - role_parameters = submit_dict["component_parameters"]["role"][role] - input_parameters[role] = [copy.deepcopy(cpn_dict)] * len(submit_dict["role"][role]) - - for idx, parameters in role_parameters.items(): - for reader in components: - if reader not in parameters: - continue - - if idx == "all": - party_id_list = submit_dict["role"][role] - for i in range(len(party_id_list)): - input_parameters[role][i][reader] = parameters[reader] - elif len(idx.split("|")) == 1: - input_parameters[role][int(idx)][reader] = parameters[reader] - else: - id_set = list(map(int, idx.split("|"))) - for _id in id_set: - input_parameters[role][_id][reader] = parameters[reader] - - return input_parameters - - @staticmethod - def get_job_parameters(submit_dict): - ret = {} - job_parameters = submit_dict.get("job_parameters", {}) - component_parameters = submit_dict.get("component_parameters", {}) - common_job_parameters = job_parameters.get("common", {}) - role_job_parameters = component_parameters.get("role", {}) - for role in submit_dict["role"]: - party_id_list = submit_dict["role"][role] - if not role_job_parameters: - ret[role] = {party_id: copy.deepcopy(common_job_parameters) for party_id in party_id_list} - continue - - ret[role] = {} - for idx in range(len(party_id_list)): - role_ids = role_job_parameters.get(role, {}).keys() - parameters = copy.deepcopy(common_job_parameters) - for role_id in role_ids: - if role_id == "all" or str(idx) in role_id.split("|"): - parameters = RuntimeConfParserUtil.merge_dict(parameters, - role_job_parameters.get(role, {})[role_id]) - - ret[role][party_id_list[idx]] = parameters - - return ret - - @staticmethod - def generate_predict_conf_template(predict_dsl, train_conf, model_id, model_version): - if not train_conf.get("role") or not train_conf.get("initiator"): - raise ValueError("role and initiator should be contain in job's trainconf") - - predict_conf = dict() - predict_conf["dsl_version"] = 2 - predict_conf["role"] = train_conf.get("role") - predict_conf["initiator"] = train_conf.get("initiator") - - predict_conf["job_parameters"] = train_conf.get("job_parameters", {}) - predict_conf["job_parameters"]["common"].update({"model_id": model_id, - "model_version": model_version, - "job_type": "predict"}) - - predict_conf["component_parameters"] = {"role": {}} - - for role in predict_conf["role"]: - if role not in ["guest", "host"]: - continue - - reader_components = [] - for module_alias, module_info in predict_dsl.get("components", {}).items(): - if module_info["module"] == "Reader": - reader_components.append(module_alias) - - predict_conf["component_parameters"]["role"][role] = dict() - fill_template = {} - for idx, reader_alias in enumerate(reader_components): - fill_template[reader_alias] = {"table": {"name": "name_to_be_filled_" + str(idx), - "namespace": "namespace_to_be_filled_" + str(idx)}} - - for idx in range(len(predict_conf["role"][role])): - predict_conf["component_parameters"]["role"][role][str(idx)] = fill_template - - return predict_conf diff --git a/python/fate_flow/utils/schedule_utils.py b/python/fate_flow/utils/schedule_utils.py index 3bef373ec..5c82190c1 100644 --- a/python/fate_flow/utils/schedule_utils.py +++ b/python/fate_flow/utils/schedule_utils.py @@ -13,172 +13,50 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import typing -from functools import wraps - -from fate_arch.common.base_utils import current_timestamp -from fate_flow.db.db_models import DB, Job -from fate_flow.scheduler.dsl_parser import DSLParserV1, DSLParserV2 -from fate_flow.utils.config_adapter import JobRuntimeConfigAdapter +from fate_flow.db.base_models import DB +from fate_flow.db.schedule_models import ScheduleJob +from fate_flow.utils.base_utils import current_timestamp from fate_flow.utils.log_utils import schedule_logger -@DB.connection_context() -def ready_signal(job_id, set_or_reset: bool, ready_timeout_ttl=None): - filters = [Job.f_job_id == job_id] - if set_or_reset: - update_fields = {Job.f_ready_signal: True, Job.f_ready_time: current_timestamp()} - filters.append(Job.f_ready_signal == False) - else: - update_fields = {Job.f_ready_signal: False, Job.f_ready_time: None} - filters.append(Job.f_ready_signal == True) - if ready_timeout_ttl: - filters.append(current_timestamp() - Job.f_ready_time > ready_timeout_ttl) - update_status = Job.update(update_fields).where(*filters).execute() > 0 - return update_status - - @DB.connection_context() def cancel_signal(job_id, set_or_reset: bool): - update_status = Job.update({Job.f_cancel_signal: set_or_reset, Job.f_cancel_time: current_timestamp()}).where(Job.f_job_id == job_id).execute() > 0 + update_status = ScheduleJob.update({ScheduleJob.f_cancel_signal: set_or_reset, ScheduleJob.f_cancel_time: current_timestamp()}).where(ScheduleJob.f_job_id == job_id).execute() > 0 return update_status @DB.connection_context() def rerun_signal(job_id, set_or_reset: bool): if set_or_reset is True: - update_fields = {Job.f_rerun_signal: True, Job.f_cancel_signal: False, Job.f_end_scheduling_updates: 0} + update_fields = {ScheduleJob.f_rerun_signal: True, ScheduleJob.f_cancel_signal: False, ScheduleJob.f_end_scheduling_updates: 0} elif set_or_reset is False: - update_fields = {Job.f_rerun_signal: False} + update_fields = {ScheduleJob.f_rerun_signal: False} else: raise RuntimeError(f"can not support rereun signal {set_or_reset}") - update_status = Job.update(update_fields).where(Job.f_job_id == job_id).execute() > 0 + update_status = ScheduleJob.update(update_fields).where(ScheduleJob.f_job_id == job_id).execute() > 0 return update_status -def schedule_lock(func): - @wraps(func) - def _wrapper(*args, **kwargs): - _lock = kwargs.pop("lock", False) - if _lock: - job = kwargs.get("job") - schedule_logger(job.f_job_id).info(f"get job {job.f_job_id} schedule lock") - _result = None - if not ready_signal(job_id=job.f_job_id, set_or_reset=True): - schedule_logger(job.f_job_id).info(f"get job {job.f_job_id} schedule lock failed, job may be handled by another scheduler") - return - try: - _result = func(*args, **kwargs) - except Exception as e: - raise e - finally: - ready_signal(job_id=job.f_job_id, set_or_reset=False) - schedule_logger(job.f_job_id).info(f"release job {job.f_job_id} schedule lock") - return _result - else: - return func(*args, **kwargs) - return _wrapper - - @DB.connection_context() -def get_job_dsl_parser_by_job_id(job_id): - jobs = Job.select(Job.f_dsl, Job.f_runtime_conf_on_party, Job.f_train_runtime_conf).where(Job.f_job_id == job_id) - if jobs: - job = jobs[0] - job_dsl_parser = get_job_dsl_parser(dsl=job.f_dsl, runtime_conf=job.f_runtime_conf_on_party, - train_runtime_conf=job.f_train_runtime_conf) - return job_dsl_parser, job.f_runtime_conf_on_party, job.f_dsl - else: - return None, None, None - - -def get_conf_version(conf: dict): - return int(conf.get("dsl_version", "1")) - - -def get_job_dsl_parser(dsl=None, runtime_conf=None, pipeline_dsl=None, train_runtime_conf=None, job_type=None): - parser_version = get_conf_version(runtime_conf) - - if parser_version == 1: - dsl, runtime_conf = convert_dsl_and_conf_v1_to_v2(dsl, runtime_conf) - if pipeline_dsl and train_runtime_conf: - pipeline_dsl, train_runtime_conf = convert_dsl_and_conf_v1_to_v2(pipeline_dsl, train_runtime_conf) - parser_version = 2 - - dsl_parser = get_dsl_parser_by_version(parser_version) - if not job_type: - job_type = JobRuntimeConfigAdapter(runtime_conf).get_job_type() - dsl_parser.run(dsl=dsl, - runtime_conf=runtime_conf, - pipeline_dsl=pipeline_dsl, - pipeline_runtime_conf=train_runtime_conf, - mode=job_type) - return dsl_parser - - -def federated_order_reset(dest_parties, scheduler_partys_info): - dest_partys_new = [] - scheduler = [] - dest_party_ids_dict = {} - for dest_role, dest_party_ids in dest_parties: - from copy import deepcopy - new_dest_party_ids = deepcopy(dest_party_ids) - dest_party_ids_dict[dest_role] = new_dest_party_ids - for scheduler_role, scheduler_party_id in scheduler_partys_info: - if dest_role == scheduler_role and scheduler_party_id in dest_party_ids: - dest_party_ids_dict[dest_role].remove(scheduler_party_id) - scheduler.append((scheduler_role, [scheduler_party_id])) - if dest_party_ids_dict[dest_role]: - dest_partys_new.append((dest_role, dest_party_ids_dict[dest_role])) - if scheduler: - dest_partys_new.extend(scheduler) - return dest_partys_new - - -def get_parser_version_mapping(): - return { - "1": DSLParserV1(), - "2": DSLParserV2() - } - - -def get_dsl_parser_by_version(version: typing.Union[str, int] = 2): - mapping = get_parser_version_mapping() - if isinstance(version, int): - version = str(version) - if version not in mapping: - raise Exception("{} version of dsl parser is not currently supported.".format(version)) - return mapping[version] - - -def fill_inference_dsl(dsl_parser: DSLParserV2, origin_inference_dsl, components_parameters: dict = None): - # must fill dsl for fate serving - if isinstance(dsl_parser, DSLParserV2): - components_module_name = {} - for component, param in components_parameters.items(): - components_module_name[component] = param["CodePath"] - return dsl_parser.get_predict_dsl(predict_dsl=origin_inference_dsl, module_object_dict=components_module_name) +def schedule_signal(job_id: object, set_or_reset: bool) -> bool: + filters = [ScheduleJob.f_job_id == job_id] + if set_or_reset: + update_fields = {ScheduleJob.f_schedule_signal: True, ScheduleJob.f_schedule_time: current_timestamp()} + filters.append(ScheduleJob.f_schedule_signal == False) else: - raise Exception(f"not support dsl parser {type(dsl_parser)}") - - -def convert_dsl_and_conf_v1_to_v2(dsl, runtime_conf): - dsl_parser_v1 = DSLParserV1() - dsl = dsl_parser_v1.convert_dsl_v1_to_v2(dsl) - components = dsl_parser_v1.get_components_light_weight(dsl) - - from fate_flow.db.component_registry import ComponentRegistry - job_providers = dsl_parser_v1.get_job_providers(dsl=dsl, provider_detail=ComponentRegistry.REGISTRY) - cpn_role_parameters = dict() + update_fields = {ScheduleJob.f_schedule_signal: False, ScheduleJob.f_schedule_time: None} + filters.append(ScheduleJob.f_schedule_signal == True) + update_status = ScheduleJob.update(update_fields).where(*filters).execute() > 0 + if set_or_reset and not update_status: + # update timeout signal + schedule_timeout_signal(job_id) + return update_status - for cpn in components: - cpn_name = cpn.get_name() - role_params = dsl_parser_v1.parse_component_role_parameters( - component=cpn_name, dsl=dsl, runtime_conf=runtime_conf, - provider_detail=ComponentRegistry.REGISTRY, - provider_name=job_providers[cpn_name]["provider"]["name"], - provider_version=job_providers[cpn_name]["provider"]["version"]) - cpn_role_parameters[cpn_name] = role_params - runtime_conf = dsl_parser_v1.convert_conf_v1_to_v2(runtime_conf, cpn_role_parameters) - return dsl, runtime_conf +def schedule_timeout_signal(job_id, ready_timeout_ttl: int = 10*6000): + job_list = ScheduleJob.query(job_id=job_id, schedule_signal=True) + if job_list: + job = job_list[0] + if current_timestamp() - job.f_schedule_time > ready_timeout_ttl: + schedule_logger(job_id).info("schedule timeout, try to update signal") + schedule_signal(job_id, set_or_reset=False) diff --git a/python/fate_flow/utils/session_utils.py b/python/fate_flow/utils/session_utils.py deleted file mode 100644 index ff8a9ec49..000000000 --- a/python/fate_flow/utils/session_utils.py +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse - -from fate_flow.utils.log_utils import schedule_logger -from fate_arch import session - - -class SessionStop(object): - @classmethod - def run(cls): - parser = argparse.ArgumentParser() - parser.add_argument('--session', required=True, type=str, help="session manager id") - parser.add_argument('--computing', help="computing engine", type=str) - parser.add_argument('--federation', help="federation engine", type=str) - parser.add_argument('--storage', help="storage engine", type=str) - parser.add_argument('-c', '--command', required=True, type=str, help="command") - args = parser.parse_args() - session_id = args.session - fate_job_id = session_id.split('_')[0] - with session.Session(session_id=session_id, - options={"logger": schedule_logger(fate_job_id)}) as sess: - sess.destroy_all_sessions() - - -if __name__ == '__main__': - SessionStop.run() diff --git a/python/fate_flow/utils/task_utils.py b/python/fate_flow/utils/task_utils.py deleted file mode 100644 index 7ec01e7b9..000000000 --- a/python/fate_flow/utils/task_utils.py +++ /dev/null @@ -1,43 +0,0 @@ -import functools - -from flask import request as flask_request - -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.entity import RetCode -from fate_flow.operation.job_saver import JobSaver -from fate_flow.utils.api_utils import get_json_result -from fate_flow.utils.requests_utils import request - - -def task_request_proxy(filter_local=False, force=True): - def _outer(func): - @functools.wraps(func) - def _wrapper(*args, **kwargs): - party_id, role, task_id, task_version = kwargs.get("party_id"), kwargs.get("role"), \ - kwargs.get("task_id"), kwargs.get("task_version") - if not filter_local or (filter_local and role == "local"): - tasks = JobSaver.query_task(task_id=task_id, task_version=task_version, role=role, party_id=party_id) - if tasks: - if tasks[0].f_run_ip and tasks[0].f_run_port: - if tasks[0].f_run_ip != RuntimeConfig.JOB_SERVER_HOST: - source_url = flask_request.url - source_address = source_url.split("/")[2] - dest_address = ":".join([tasks[0].f_run_ip, str(tasks[0].f_run_port)]) - dest_url = source_url.replace(source_address, dest_address) - try: - response = request(method=flask_request.method, url=dest_url, json=flask_request.json, headers=flask_request.headers) - if 200 <= response.status_code < 300: - response = response.json() - return get_json_result(retcode=response.get("retcode"), - retmsg=response.get('retmsg')) - else: - raise Exception(f"status_code: {response.status_code}, text: {response.text}") - except Exception as e: - if force: - return func(*args, **kwargs) - raise e - else: - return get_json_result(retcode=RetCode.DATA_ERROR, retmsg='no found task') - return func(*args, **kwargs) - return _wrapper - return _outer diff --git a/python/fate_flow/utils/upload_utils.py b/python/fate_flow/utils/upload_utils.py deleted file mode 100644 index fff07996e..000000000 --- a/python/fate_flow/utils/upload_utils.py +++ /dev/null @@ -1,103 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse -import uuid - -from fate_arch import storage -from fate_arch.session import Session -from fate_arch.storage import StorageEngine, EggRollStoreType, StorageTableOrigin -from fate_flow.utils import data_utils - - -class UploadFile(object): - @classmethod - def run(cls): - parser = argparse.ArgumentParser() - parser.add_argument('--session_id', required=True, type=str, help="session id") - parser.add_argument('--storage', help="storage engine", type=str) - parser.add_argument('--file', required=True, type=str, help="file path") - parser.add_argument('--namespace', required=True, type=str, help="namespace") - parser.add_argument('--name', required=True, type=str, help="name") - parser.add_argument('--partitions', required=True, type=int, help="partitions") - args = parser.parse_args() - session_id = args.session_id - with Session(session_id=session_id) as sess: - storage_session = sess.storage( - storage_engine=args.storage - ) - if args.storage in {StorageEngine.EGGROLL, StorageEngine.STANDALONE}: - upload_address = { - "name": args.name, - "namespace": args.namespace, - "storage_type": EggRollStoreType.ROLLPAIR_LMDB, - } - address = storage.StorageTableMeta.create_address( - storage_engine=args.storage, address_dict=upload_address - ) - table = storage_session.create_table(address=address, name=args.name, namespace=args.namespace, partitions=args.partitions, origin=StorageTableOrigin.UPLOAD) - cls.upload(args.file, False, table=table) - - @classmethod - def upload(cls, input_file, head, table=None, id_delimiter=",", extend_sid=False): - with open(input_file, "r") as fin: - if head is True: - data_head = fin.readline() - _, meta = table.meta.update_metas( - schema=data_utils.get_header_schema( - header_line=data_head, - id_delimiter=id_delimiter - ) - ) - table.meta = meta - fate_uuid = uuid.uuid1().hex - get_line = cls.get_line(extend_sid) - line_index = 0 - n = 0 - while True: - data = list() - lines = fin.readlines(1024 * 1024 * 8 * 500) - if lines: - # self.append_data_line(lines, data, n) - for line in lines: - values = line.rstrip().split(',') - k, v = get_line( - values=values, - line_index=line_index, - extend_sid=extend_sid, - auto_increasing_sid=False, - id_delimiter=id_delimiter, - fate_uuid=fate_uuid - ) - data.append((k, v)) - line_index += 1 - table.put_all(data) - if n == 0: - table.meta.update_metas(part_of_data=data[:100]) - n += 1 - else: - return line_index - - @classmethod - def get_line(self, extend_sid=False): - if extend_sid: - line = data_utils.get_sid_data_line - else: - line = data_utils.get_data_line - return line - - -if __name__ == '__main__': - UploadFile.run() diff --git a/python/fate_flow/apps/worker_app.py b/python/fate_flow/utils/version.py similarity index 59% rename from python/fate_flow/apps/worker_app.py rename to python/fate_flow/utils/version.py index adecd4565..192de98f3 100644 --- a/python/fate_flow/apps/worker_app.py +++ b/python/fate_flow/utils/version.py @@ -13,20 +13,23 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import os.path +import os -from flask import request +import dotenv +import typing -from fate_arch.common.file_utils import load_json_conf -from fate_flow.utils.api_utils import get_json_result +from fate_flow.runtime.system_settings import VERSION_FILE_PATH -page_name = "worker" +def get_versions() -> typing.Mapping[str, typing.Any]: + return dotenv.dotenv_values( + dotenv_path=VERSION_FILE_PATH + ) -@manager.route('/config/load', methods=['POST']) -def load_config(): - conf_path = request.json.get('config_path') - data = {} - if os.path.exists(conf_path): - data = load_json_conf(conf_path) - return get_json_result(data=data) + +def get_flow_version() -> typing.Optional[str]: + return get_versions().get("FATE_FLOW") + + +def get_default_fate_version() -> typing.Optional[str]: + return get_versions().get("FATE") diff --git a/python/fate_flow/utils/wraps_utils.py b/python/fate_flow/utils/wraps_utils.py new file mode 100644 index 000000000..a492a5b2d --- /dev/null +++ b/python/fate_flow/utils/wraps_utils.py @@ -0,0 +1,239 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import threading +from functools import wraps + +from fate_flow.entity.code import ReturnCode + +from flask import request as flask_request +from fate_flow.errors.server_error import NoFoundTask, ResponseException, NoFoundINSTANCE, NoPermission +from fate_flow.manager.operation.job_saver import JobSaver +from fate_flow.runtime.runtime_config import RuntimeConfig +from fate_flow.runtime.system_settings import HOST, HTTP_PORT, API_VERSION +from fate_flow.utils.api_utils import API, federated_coordination_on_http +from fate_flow.utils.log_utils import schedule_logger +from fate_flow.utils.requests_utils import request +from fate_flow.utils.schedule_utils import schedule_signal +from fate_flow.db.casbin_models import FATE_CASBIN + + +def filter_parameters(filter_value=None): + def _inner(func): + @wraps(func) + def _wrapper(*args, **kwargs): + _kwargs = {} + for k, v in kwargs.items(): + if v != filter_value: + _kwargs[k] = v + return func(*args, **_kwargs) + return _wrapper + return _inner + + +def switch_function(switch, code=ReturnCode.Server.FUNCTION_RESTRICTED, message="function restricted"): + def _inner(func): + @wraps(func) + def _wrapper(*args, **kwargs): + if switch: + return func(*args, **kwargs) + else: + raise Exception(code, f"func {func.__name__}, {message}") + return _wrapper + return _inner + + +def task_request_proxy(filter_local=False, force=True): + def _outer(func): + @wraps(func) + def _wrapper(*args, **kwargs): + party_id, role, task_id, task_version = kwargs.get("party_id"), kwargs.get("role"), \ + kwargs.get("task_id"), kwargs.get("task_version") + if not filter_local or (filter_local and role == "local"): + tasks = JobSaver.query_task(task_id=task_id, task_version=task_version, role=role, party_id=party_id) + if tasks: + if tasks[0].f_run_ip and tasks[0].f_run_port: + if tasks[0].f_run_ip != RuntimeConfig.JOB_SERVER_HOST: + source_url = flask_request.url + source_address = source_url.split("/")[2] + dest_address = ":".join([tasks[0].f_run_ip, str(tasks[0].f_run_port)]) + dest_url = source_url.replace(source_address, dest_address) + try: + response = request(method=flask_request.method, url=dest_url, json=flask_request.json, + headers=flask_request.headers, params=flask_request.args) + if 200 <= response.status_code < 300: + response = response.json() + return API.Output.json(code=response.get("code"), message=response.get("message")) + else: + raise ResponseException(response=response.text) + except Exception as e: + if force: + return func(*args, **kwargs) + raise e + else: + return API.Output.fate_flow_exception(NoFoundTask( + role=role, + party_id=party_id, + task_id=task_id, + task_version=task_version + )) + return func(*args, **kwargs) + return _wrapper + return _outer + + +def cluster_route(func): + @wraps(func) + def _route(*args, **kwargs): + instance_id = kwargs.get('instance_id') + request_data = flask_request.json or flask_request.form.to_dict() + if not instance_id: + return func(*args, **kwargs) + instance = RuntimeConfig.SERVICE_DB.get_servers().get(instance_id) + if instance is None: + return API.Output.fate_flow_exception(NoFoundINSTANCE(instance_id=instance_id)) + + if instance.http_address == f'{HOST}:{HTTP_PORT}': + return func(*args, **kwargs) + + endpoint = flask_request.full_path + prefix = f'/{API_VERSION}/' + if endpoint.startswith(prefix): + endpoint = endpoint[len(prefix) - 1:] + response = federated_coordination_on_http( + method=flask_request.method, + host=instance.host, + port=instance.http_port, + endpoint=endpoint, + json_body=request_data, + headers=flask_request.headers, + ) + return API.Output.json(**response) + return _route + + +def schedule_lock(func): + @wraps(func) + def _wrapper(*args, **kwargs): + _lock = kwargs.pop("lock", False) + if _lock: + job = kwargs.get("job") + schedule_logger(job.f_job_id).debug(f"get job {job.f_job_id} schedule lock") + _result = None + if not schedule_signal(job_id=job.f_job_id, set_or_reset=True): + schedule_logger(job.f_job_id).warn(f"get job {job.f_job_id} schedule lock failed, " + f"job may be handled by another scheduler") + return + try: + _result = func(*args, **kwargs) + except Exception as e: + schedule_logger(job.f_job_id).exception(e) + raise e + finally: + schedule_signal(job_id=job.f_job_id, set_or_reset=False) + schedule_logger(job.f_job_id).debug(f"release job {job.f_job_id} schedule lock") + return _result + else: + return func(*args, **kwargs) + return _wrapper + + +def threading_lock(func): + @wraps(func) + def _wrapper(*args, **kwargs): + with threading.Lock(): + return func(*args, **kwargs) + return _wrapper + + +def asynchronous_function(func): + @wraps(func) + def _wrapper(*args, **kwargs): + is_asynchronous = kwargs.pop("is_asynchronous", False) + if is_asynchronous: + thread = threading.Thread(target=func, args=args, kwargs=kwargs) + thread.start() + is_asynchronous = True + return is_asynchronous + else: + return func(*args, **kwargs) + return _wrapper + + +def check_permission(operate=None, types=None): + def _inner(func): + @wraps(func) + def _wrapper(*args, **kwargs): + _init = kwargs.get("init", False) + if not _init: + conf_app_id = flask_request.headers.get("Appid") + conf_roles_dct = [roles for roles in FATE_CASBIN.get_roles_for_user(conf_app_id)] + if conf_app_id == "admin": + conf_role = conf_app_id + elif len(conf_roles_dct): + if "super_client" in conf_roles_dct: + conf_role = "super_client" + else: + conf_role = "client" + else: + raise NoPermission + if types == "client": + app_id = kwargs.get("app_id") + if app_id != "admin": + app_id_role = "super_client" if FATE_CASBIN.has_role_for_user(app_id, "super_client") else "client" + else: + app_id_role = "admin" + if operate == "query": + if conf_role == "super_client": + if conf_app_id != app_id: + if app_id_role != "client": + raise NoPermission + if conf_role == "client" and conf_app_id != app_id: + raise NoPermission + if operate == "delete" and ( + app_id == conf_app_id + or (conf_role == "super_client" and app_id_role in ["admin", "super_client"]) + or conf_role == "client"): + raise NoPermission + if operate == "create" and conf_role == "client": raise NoPermission + + if types == "permission": + app_id = kwargs.get("app_id") + if app_id != "admin": + app_id_role = FATE_CASBIN.has_role_for_user(app_id, "super_client") + app_id_role = "super_client" if app_id_role else "client" + else: + app_id_role = "admin" + if operate == "query" and conf_role == "client" and conf_app_id != app_id: + raise NoPermission + if operate == "grant": + grant_role = kwargs.get("grant_role", False) + if not grant_role: + grant_role = kwargs.get("role", False) + if conf_role == "admin" and app_id_role == "admin": raise NoPermission + if conf_role == "super_client": + app_id_role_client = FATE_CASBIN.get_roles_for_user(app_id) + if app_id_role_client: raise NoPermission + if grant_role == "super_client": raise NoPermission + if conf_role == "client": raise NoPermission + if operate == "delete": + grant_role = kwargs.get("grant_role", None) + if grant_role and conf_role == "super_client" and grant_role == "super_client":raise NoPermission + if conf_role == app_id_role: raise NoPermission + if conf_role == "super_client" and app_id_role != "client":raise NoPermission + if conf_role == "client":raise NoPermission + return func(*args, **kwargs) + return _wrapper + return _inner diff --git a/python/fate_flow/utils/xthread.py b/python/fate_flow/utils/xthread.py index e806fc4d5..e17c56c58 100644 --- a/python/fate_flow/utils/xthread.py +++ b/python/fate_flow/utils/xthread.py @@ -1,3 +1,17 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # Copyright 2009 Brian Quinlan. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. @@ -12,7 +26,10 @@ import threading import weakref import os -from fate_flow.settings import stat_logger + +from fate_flow.utils.log import getLogger + +stat_logger = getLogger() # Workers are created as daemon threads. This is done to allow the interpreter # to exit when there are still idle threads in a ThreadPoolExecutor's thread @@ -40,8 +57,10 @@ def _python_exit(): for t, q in items: t.join() + atexit.register(_python_exit) + class _WorkItem(object): def __init__(self, future, fn, args, kwargs): self.future = future @@ -62,6 +81,7 @@ def run(self): else: self.future.set_result(result) + def _worker(executor_reference, work_queue): try: while True: @@ -85,8 +105,8 @@ def _worker(executor_reference, work_queue): except BaseException: _base.LOGGER.critical('Exception in worker', exc_info=True) -class ThreadPoolExecutor(_base.Executor): +class ThreadPoolExecutor(_base.Executor): # Used to assign unique thread names when thread_name_prefix is not supplied. _counter = itertools.count().__next__ @@ -113,6 +133,10 @@ def __init__(self, max_workers=None, thread_name_prefix=''): self._thread_name_prefix = (thread_name_prefix or ("ThreadPoolExecutor-%d" % self._counter())) + @property + def max_workers(self): + return self._max_workers + def submit(self, fn, *args, **kwargs): with self._shutdown_lock: if self._shutdown: diff --git a/python/fate_flow/worker/base_worker.py b/python/fate_flow/worker/base_worker.py deleted file mode 100644 index 9949f3729..000000000 --- a/python/fate_flow/worker/base_worker.py +++ /dev/null @@ -1,179 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import argparse -import os -import sys -import traceback -import logging - -import requests - -from fate_arch.common.base_utils import current_timestamp -from fate_arch.common.file_utils import load_json_conf, dump_json_conf -from fate_flow.settings import REMOTE_LOAD_CONF -from fate_flow.utils.log_utils import getLogger, LoggerFactory, exception_to_trace_string -from fate_flow.db.component_registry import ComponentRegistry -from fate_flow.db.config_manager import ConfigManager -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.entity.types import ProcessRole -from fate_flow.entity import BaseEntity - -LOGGER = getLogger() - - -class WorkerArgs(BaseEntity): - def __init__(self, **kwargs): - self.job_id = kwargs.get("job_id") - self.component_name = kwargs.get("component_name") - self.task_id = kwargs.get("task_id") - self.task_version = kwargs.get("task_version") - self.role = kwargs.get("role") - self.party_id = kwargs.get("party_id") - self.config = self.load_dict_attr(kwargs, "config") - self.result = kwargs.get("result") - self.log_dir = kwargs.get("log_dir") - self.parent_log_dir = kwargs.get("parent_log_dir") - - self.worker_id = kwargs.get("worker_id") - self.run_ip = kwargs.get("run_ip") - self.run_port = kwargs.get("run_port") - self.job_server = kwargs.get("job_server") - - # TaskInitializer - self.result = kwargs.get("result") - self.dsl = self.load_dict_attr(kwargs, "dsl") - self.runtime_conf = self.load_dict_attr(kwargs, "runtime_conf") - self.train_runtime_conf = self.load_dict_attr(kwargs, "train_runtime_conf") - self.pipeline_dsl = self.load_dict_attr(kwargs, "pipeline_dsl") - - # TaskSender & TaskReceiver - self.session_id = kwargs.get("session_id") - self.federation_session_id = kwargs.get("federation_session_id") - - # TaskSender - self.receive_topic = kwargs.get("receive_topic") - - # TaskReceiver - self.http_port = kwargs.get("http_port") - self.grpc_port = kwargs.get("grpc_port") - - # Dependence Upload - self.dependence_type = kwargs.get("dependence_type") - - self.is_deepspeed = kwargs.get("is_deepspeed") - self.model_path = kwargs.get("model_path") - self.computing_engine = kwargs.get("computing_engine") - - @staticmethod - def load_dict_attr(kwargs: dict, attr_name: str): - if kwargs.get(attr_name): - if REMOTE_LOAD_CONF and kwargs.get("is_deepspeed"): - url = f'http://{kwargs.get("job_server")}/v1/worker/config/load' - try: - _r = requests.post(url, json={"config_path": kwargs[attr_name]}).json() - config = _r.get("data") if _r.get("data") else {} - except Exception as e: - LOGGER.exception(e) - config = {} - else: - config = load_json_conf(kwargs[attr_name]) - else: - config = {} - return config - - -class BaseWorker: - def __init__(self): - self.args: WorkerArgs = None - self.run_pid = None - self.report_info = {} - - def run(self, **kwargs): - result = {} - code = 0 - message = "" - start_time = current_timestamp() - self.run_pid = os.getpid() - try: - self.args = self.get_args(**kwargs) - if self.args.model_path: - os.environ["MODEL_PATH"] = self.args.model_path - RuntimeConfig.init_env() - role = ProcessRole(os.getenv("PROCESS_ROLE")) - append_to_parent_log = True - if self.args.is_deepspeed: - role = ProcessRole(ProcessRole.WORKER.value) - append_to_parent_log = False - RuntimeConfig.set_process_role(role) - if RuntimeConfig.PROCESS_ROLE == ProcessRole.WORKER: - LoggerFactory.LEVEL = logging.getLevelName(os.getenv("FATE_LOG_LEVEL", "INFO")) - if os.getenv("EGGROLL_CONTAINER_LOGS_DIR"): - # eggroll deepspeed - self.args.parent_log_dir = os.path.dirname(os.getenv("EGGROLL_CONTAINER_LOGS_DIR")) - self.args.log_dir = os.getenv("EGGROLL_CONTAINER_LOGS_DIR") - LoggerFactory.set_directory(directory=self.args.log_dir, parent_log_dir=self.args.parent_log_dir, - append_to_parent_log=append_to_parent_log, force=True) - LOGGER.info(f"enter {self.__class__.__name__} worker in subprocess, pid: {self.run_pid}") - else: - LOGGER.info(f"enter {self.__class__.__name__} worker in driver process, pid: {self.run_pid}") - LOGGER.info(f"log level: {logging.getLevelName(LoggerFactory.LEVEL)}") - for env in {"VIRTUAL_ENV", "PYTHONPATH", "SPARK_HOME", "FATE_DEPLOY_BASE", "PROCESS_ROLE", "FATE_JOB_ID"}: - LOGGER.info(f"{env}: {os.getenv(env)}") - if self.args.job_server: - RuntimeConfig.init_config(JOB_SERVER_HOST=self.args.job_server.split(':')[0], - HTTP_PORT=self.args.job_server.split(':')[1]) - if not RuntimeConfig.LOAD_COMPONENT_REGISTRY: - ComponentRegistry.load() - if not RuntimeConfig.LOAD_CONFIG_MANAGER: - ConfigManager.load() - result = self._run() - except Exception as e: - LOGGER.exception(e) - traceback.print_exc() - try: - self._handle_exception() - except Exception as e: - LOGGER.exception(e) - code = 1 - message = exception_to_trace_string(e) - finally: - if self.args and self.args.result: - if not self.args.is_deepspeed: - dump_json_conf(result, self.args.result) - end_time = current_timestamp() - LOGGER.info(f"worker {self.__class__.__name__}, process role: {RuntimeConfig.PROCESS_ROLE}, pid: {self.run_pid}, elapsed: {end_time - start_time} ms") - if RuntimeConfig.PROCESS_ROLE == ProcessRole.WORKER: - sys.exit(code) - if self.args and self.args.is_deepspeed: - sys.exit(code) - else: - return code, message, result - - def _run(self): - raise NotImplementedError - - def _handle_exception(self): - pass - - @staticmethod - def get_args(**kwargs): - if kwargs: - return WorkerArgs(**kwargs) - else: - parser = argparse.ArgumentParser() - for arg in WorkerArgs().to_dict(): - parser.add_argument(f"--{arg}", required=False) - return WorkerArgs(**parser.parse_args().__dict__) diff --git a/python/fate_flow/worker/dependence_upload.py b/python/fate_flow/worker/dependence_upload.py deleted file mode 100644 index 90cbf876d..000000000 --- a/python/fate_flow/worker/dependence_upload.py +++ /dev/null @@ -1,179 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import functools -import os -import shutil -import zipfile -import subprocess - -from fate_arch.common import file_utils -from fate_flow.utils.log_utils import getLogger -from fate_flow.db.db_models import ComponentProviderInfo -from fate_flow.db.dependence_registry import DependenceRegistry -from fate_flow.db.service_registry import ServerRegistry -from fate_flow.entity import ComponentProvider -from fate_flow.entity.types import FateDependenceName, ComponentProviderName, FateDependenceStorageEngine -from fate_flow.settings import FATE_VERSION_DEPENDENCIES_PATH -from fate_flow.worker.base_worker import BaseWorker -from fate_flow.utils.base_utils import get_fate_flow_python_directory - -LOGGER = getLogger() - - -def upload_except_exit(func): - @functools.wraps(func) - def _wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except Exception as e: - provider = kwargs.get("provider") - dependence_type = kwargs.get("dependence_type") - storage_engine = FateDependenceStorageEngine.HDFS.value - storage_meta = { - "f_storage_engine": storage_engine, - "f_type": dependence_type, - "f_version": provider.version, - "f_upload_status": False - } - DependenceRegistry.save_dependencies_storage_meta(storage_meta) - raise e - return _wrapper - - -class DependenceUpload(BaseWorker): - def _run(self): - provider = ComponentProvider(**self.args.config.get("provider")) - dependence_type = self.args.dependence_type - self.upload_dependencies_to_hadoop(provider=provider, dependence_type=dependence_type) - - @classmethod - @upload_except_exit - def upload_dependencies_to_hadoop(cls, provider, dependence_type, storage_engine=FateDependenceStorageEngine.HDFS.value): - LOGGER.info(f'upload {dependence_type} dependencies to hadoop') - LOGGER.info(f'dependencies loading ...') - if dependence_type == FateDependenceName.Python_Env.value: - # todo: version python env - target_file = os.path.join(FATE_VERSION_DEPENDENCIES_PATH, provider.version, "python_env.tar.gz") - venv_pack_path = os.path.join(os.getenv("VIRTUAL_ENV"), "bin/venv-pack") - subprocess.run([venv_pack_path, "-o", target_file]) - source_path = os.path.dirname(os.path.dirname(os.getenv("VIRTUAL_ENV"))) - cls.rewrite_pyvenv_cfg(os.path.join(os.getenv("VIRTUAL_ENV"), "pyvenv.cfg"), "python_env") - dependencies_conf = {"executor_python": f"./{dependence_type}/bin/python", - "driver_python": f"{os.path.join(os.getenv('VIRTUAL_ENV'), 'bin', 'python')}"} - else: - fate_code_dependencies = { - "fate_flow": get_fate_flow_python_directory("fate_flow"), - "fate_arch": file_utils.get_fate_python_directory("fate_arch"), - "conf": file_utils.get_project_base_directory("conf") - } - fate_flow_snapshot_time = DependenceRegistry.get_modify_time(fate_code_dependencies["fate_flow"]) - fate_code_base_dir = os.path.join(FATE_VERSION_DEPENDENCIES_PATH, provider.version, "fate_code", "fate") - python_base_dir = os.path.join(fate_code_base_dir, "python") - if os.path.exists(os.path.dirname(python_base_dir)): - shutil.rmtree(os.path.dirname(python_base_dir)) - for key, path in fate_code_dependencies.items(): - cls.copy_dir(path, os.path.join(python_base_dir, key)) - if key == "conf": - cls.move_dir(os.path.join(python_base_dir, key), os.path.dirname(fate_code_base_dir)) - if provider.name == ComponentProviderName.FATE.value: - source_path = provider.path - else: - source_path = ComponentProviderInfo.get_or_none( - ComponentProviderInfo.f_version == provider.version, - ComponentProviderInfo.f_provider_name == ComponentProviderName.FATE.value - ).f_path - cls.copy_dir(source_path, os.path.join(python_base_dir, "federatedml")) - target_file = os.path.join(FATE_VERSION_DEPENDENCIES_PATH, provider.version, "fate.zip") - cls.zip_dir(os.path.dirname(fate_code_base_dir), target_file) - dependencies_conf = {"executor_env_pythonpath": f"./{dependence_type}/fate/python:$PYTHONPATH"} - LOGGER.info(f'dependencies loading success') - - LOGGER.info(f'start upload') - snapshot_time = DependenceRegistry.get_modify_time(source_path) - hdfs_address = ServerRegistry.FATE_ON_SPARK.get("hdfs", {}).get("name_node") - LOGGER.info(f'hdfs address: {hdfs_address}') - storage_dir = f"/fate_dependence/{provider.version}" - os.system(f" {os.getenv('HADOOP_HOME')}/bin/hdfs dfs -mkdir -p {hdfs_address}{storage_dir}") - status = os.system(f"{os.getenv('HADOOP_HOME')}/bin/hdfs dfs -put -f {target_file} {hdfs_address}{storage_dir}") - LOGGER.info(f'upload end, status is {status}') - if status == 0: - storage_path = os.path.join(storage_dir, os.path.basename(target_file)) - storage_meta = { - "f_storage_engine": storage_engine, - "f_type": dependence_type, - "f_version": provider.version, - "f_storage_path": storage_path, - "f_snapshot_time": snapshot_time, - "f_fate_flow_snapshot_time": fate_flow_snapshot_time if dependence_type == FateDependenceName.Fate_Source_Code.value else None, - "f_dependencies_conf": {"archives": "#".join([storage_path, dependence_type])}, - "f_upload_status": False, - "f_pid": 0 - } - storage_meta["f_dependencies_conf"].update(dependencies_conf) - DependenceRegistry.save_dependencies_storage_meta(storage_meta) - else: - raise Exception(f"{os.getenv('HADOOP_HOME')}/bin/hdfs dfs -put {target_file} {storage_dir} failed status: {status}") - return storage_meta - - @classmethod - def zip_dir(cls, input_dir_path, output_zip_full_name, dir_list=None): - with zipfile.ZipFile(output_zip_full_name, "w", zipfile.ZIP_DEFLATED) as zip_object: - if not dir_list: - cls.zip_write(zip_object, input_dir_path, input_dir_path) - else: - for dir_name in dir_list: - dir_path = os.path.join(input_dir_path, dir_name) - cls.zip_write(zip_object, dir_path, input_dir_path) - - @classmethod - def zip_write(cls, zip_object, dir_path, input_dir_path): - for path, dirnames, filenames in os.walk(dir_path): - fpath = path.replace(input_dir_path, '') - for filename in filenames: - if os.path.exists(os.path.join(path, filename)): - zip_object.write(os.path.join(path, filename), os.path.join(fpath, filename)) - - @staticmethod - def copy_dir(source_path, target_path): - if os.path.exists(target_path): - shutil.rmtree(target_path) - shutil.copytree(source_path, target_path) - - @staticmethod - def move_dir(source_path, target_path): - shutil.move(source_path, target_path) - - - @classmethod - def rewrite_pyvenv_cfg(cls, file, dir_name): - import re - bak_file = file + '.bak' - shutil.copyfile(file, bak_file) - with open(file, "w") as fw: - with open(bak_file, 'r') as fr: - lines = fr.readlines() - match_str = None - for line in lines: - change_line = re.findall(".*=(.*)miniconda.*", line) - if change_line: - if not match_str: - match_str = change_line[0] - line = re.sub(match_str, f" ./{dir_name}/", line) - fw.write(line) - - -if __name__ == '__main__': - DependenceUpload().run() diff --git a/python/fate_flow/worker/download_model.py b/python/fate_flow/worker/download_model.py deleted file mode 100644 index 9c0eb5bd6..000000000 --- a/python/fate_flow/worker/download_model.py +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_flow.controller.engine_adapt import build_engine -from fate_flow.operation.job_saver import JobSaver -from fate_flow.utils.log_utils import schedule_logger -from fate_flow.worker.base_worker import BaseWorker - - -class DownloadModel(BaseWorker): - def _run(self): - deepspeed_engine = build_engine(self.args.computing_engine, True) - tasks = JobSaver.query_task( - task_id=self.args.task_id, - task_version=self.args.task_version, - job_id=self.args.job_id, - role=self.args.role, - party_id=self.args.party_id - ) - task = tasks[0] - schedule_logger(self.args.job_id).info("start download model") - deepspeed_engine.download_model(task) - schedule_logger(self.args.job_id).info("download model success") - - -if __name__ == '__main__': - DownloadModel().run() diff --git a/python/fate_flow/worker/job_inheritor.py b/python/fate_flow/worker/job_inheritor.py deleted file mode 100644 index 125d77630..000000000 --- a/python/fate_flow/worker/job_inheritor.py +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import traceback - -from fate_flow.controller.job_controller import JobController -from fate_flow.entity.run_status import JobInheritanceStatus -from fate_flow.operation.job_saver import JobSaver -from fate_flow.utils.log_utils import getLogger -from fate_flow.worker.base_worker import BaseWorker - -LOGGER = getLogger() - - -class JobInherit(BaseWorker): - def _run(self): - job = JobSaver.query_job(job_id=self.args.job_id, role=self.args.role, party_id=self.args.party_id)[0] - try: - JobController.job_reload(job) - except Exception as e: - traceback.print_exc() - JobSaver.update_job(job_info={"job_id": job.f_job_id, "role": job.f_role, "party_id": job.f_party_id, - "inheritance_status": JobInheritanceStatus.FAILED}) - LOGGER.exception(e) - - -if __name__ == '__main__': - JobInherit().run() diff --git a/python/fate_flow/worker/provider_registrar.py b/python/fate_flow/worker/provider_registrar.py deleted file mode 100644 index 518bf71bd..000000000 --- a/python/fate_flow/worker/provider_registrar.py +++ /dev/null @@ -1,36 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_arch.common.base_utils import json_dumps -from fate_flow.utils.log_utils import getLogger -from fate_flow.db.component_registry import ComponentRegistry -from fate_flow.entity import ComponentProvider -from fate_flow.settings import stat_logger -from fate_flow.worker.base_worker import BaseWorker - -LOGGER = getLogger() - - -class ProviderRegistrar(BaseWorker): - def _run(self): - provider = ComponentProvider(**self.args.config.get("provider")) - support_components = ComponentRegistry.register_provider(provider) - ComponentRegistry.register_components(provider.name, support_components) - ComponentRegistry.dump() - stat_logger.info(json_dumps(ComponentRegistry.REGISTRY, indent=4)) - - -if __name__ == '__main__': - ProviderRegistrar().run() diff --git a/python/fate_flow/worker/task_base_worker.py b/python/fate_flow/worker/task_base_worker.py deleted file mode 100644 index b5042007b..000000000 --- a/python/fate_flow/worker/task_base_worker.py +++ /dev/null @@ -1,122 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_flow.entity.run_status import TaskStatus -from fate_flow.scheduling_apps.client import ControllerClient -from fate_flow.utils.log_utils import getLogger -from fate_flow.worker.base_worker import BaseWorker - - -LOGGER = getLogger() - - -class ComponentInput: - def __init__( - self, - tracker, - checkpoint_manager, - task_version_id, - parameters, - datasets, - models, - caches, - job_parameters, - roles, - flow_feeded_parameters, - ) -> None: - self._tracker = tracker - self._checkpoint_manager = checkpoint_manager - self._task_version_id = task_version_id - self._parameters = parameters - self._datasets = datasets - self._models = models - self._caches = caches - self._job_parameters = job_parameters - self._roles = roles - self._flow_feeded_parameters = flow_feeded_parameters - - @property - def tracker(self): - return self._tracker - - @property - def task_version_id(self): - return self._task_version_id - - @property - def checkpoint_manager(self): - return self._checkpoint_manager - - @property - def parameters(self): - return self._parameters - - @property - def flow_feeded_parameters(self): - return self._flow_feeded_parameters - - @property - def roles(self): - return self._roles - - @property - def job_parameters(self): - return self._job_parameters - - @property - def datasets(self): - return self._datasets - - @property - def models(self): - return {k: v for k, v in self._models.items() if v is not None} - - @property - def caches(self): - return self._caches - - -class BaseTaskWorker(BaseWorker): - def _run(self): - self.report_info.update({ - "job_id": self.args.job_id, - "component_name": self.args.component_name, - "task_id": self.args.task_id, - "task_version": self.args.task_version, - "role": self.args.role, - "party_id": self.args.party_id, - "run_ip": self.args.run_ip, - "run_pid": self.run_pid - }) - self._run_() - - def _run_(self): - pass - - def _handle_exception(self): - self.report_info["party_status"] = TaskStatus.FAILED - self.report_task_info_to_driver() - - def report_task_info_to_driver(self): - if not self.args.is_deepspeed: - LOGGER.info("report {} {} {} {} {} to driver:\n{}".format( - self.__class__.__name__, - self.report_info["task_id"], - self.report_info["task_version"], - self.report_info["role"], - self.report_info["party_id"], - self.report_info - )) - ControllerClient.report_task(self.report_info) diff --git a/python/fate_flow/worker/task_executor.py b/python/fate_flow/worker/task_executor.py deleted file mode 100644 index 13e126f57..000000000 --- a/python/fate_flow/worker/task_executor.py +++ /dev/null @@ -1,467 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import importlib -import os -import sys -import traceback - -from fate_arch import session, storage -from fate_arch.common import EngineType, profile -from fate_arch.common.base_utils import current_timestamp, json_dumps -from fate_arch.computing import ComputingEngine - -from fate_flow.component_env_utils import provider_utils -from fate_flow.db.component_registry import ComponentRegistry -from fate_flow.db.db_models import TrackingOutputDataInfo, fill_db_model_object -from fate_flow.db.runtime_config import RuntimeConfig -from fate_flow.entity import DataCache, RunParameters -from fate_flow.entity.run_status import TaskStatus -from fate_flow.errors import PassError -from fate_flow.hook import HookManager -from fate_flow.manager.data_manager import DataTableTracker -from fate_flow.manager.provider_manager import ProviderManager -from fate_flow.model.checkpoint import CheckpointManager -from fate_flow.operation.job_tracker import Tracker -from fate_flow.scheduling_apps.client import TrackerClient -from fate_flow.settings import ERROR_REPORT, ERROR_REPORT_WITH_PATH -from fate_flow.utils import job_utils, schedule_utils -from fate_flow.utils.base_utils import get_fate_flow_python_directory -from fate_flow.utils.log_utils import getLogger, replace_ip -from fate_flow.utils.model_utils import gen_party_model_id -from fate_flow.worker.task_base_worker import BaseTaskWorker, ComponentInput - - -LOGGER = getLogger() - - -class TaskExecutor(BaseTaskWorker): - def _run_(self): - # todo: All function calls where errors should be thrown - args = self.args - start_time = current_timestamp() - try: - LOGGER.info(f'run {args.component_name} {args.task_id} {args.task_version} on {args.role} {args.party_id} task') - HookManager.init() - self.report_info.update({ - "job_id": args.job_id, - "component_name": args.component_name, - "task_id": args.task_id, - "task_version": args.task_version, - "role": args.role, - "party_id": args.party_id, - "run_ip": args.run_ip, - "run_port": args.run_port, - "run_pid": self.run_pid - }) - job_configuration = job_utils.get_job_configuration( - job_id=self.args.job_id, - role=self.args.role, - party_id=self.args.party_id - ) - task_parameters_conf = args.config - dsl_parser = schedule_utils.get_job_dsl_parser(dsl=job_configuration.dsl, - runtime_conf=job_configuration.runtime_conf, - train_runtime_conf=job_configuration.train_runtime_conf, - pipeline_dsl=None) - - job_parameters = dsl_parser.get_job_parameters(job_configuration.runtime_conf, int(job_configuration.runtime_conf.get("dsl_version", "1"))) - user_name = job_parameters.get(args.role, {}).get(args.party_id, {}).get("user", '') - LOGGER.info(f"user name:{user_name}") - task_parameters = RunParameters(**task_parameters_conf) - job_parameters = task_parameters - if job_parameters.assistant_role: - TaskExecutor.monkey_patch() - - job_args_on_party = TaskExecutor.get_job_args_on_party(dsl_parser, job_configuration.runtime_conf_on_party, args.role, args.party_id) - component = dsl_parser.get_component_info(component_name=args.component_name) - module_name = component.get_module() - task_input_dsl = component.get_input() - task_output_dsl = component.get_output() - - party_model_id = gen_party_model_id(job_parameters.model_id, args.role, args.party_id) - model_version = job_parameters.model_version if job_parameters.job_type != 'predict' else args.job_id - - kwargs = { - 'job_id': args.job_id, - 'role': args.role, - 'party_id': args.party_id, - 'component_name': args.component_name, - 'task_id': args.task_id, - 'task_version': args.task_version, - 'model_id': job_parameters.model_id, - # in the prediction job, job_parameters.model_version comes from the training job - # TODO: prediction job should not affect training job - 'model_version': job_parameters.model_version, - 'component_module_name': module_name, - 'job_parameters': job_parameters, - } - tracker = Tracker(**kwargs) - tracker_client = TrackerClient(**kwargs) - checkpoint_manager = CheckpointManager(**kwargs) - - predict_tracker_client = None - if job_parameters.job_type == 'predict': - kwargs['model_version'] = model_version - predict_tracker_client = TrackerClient(**kwargs) - - self.report_info["party_status"] = TaskStatus.RUNNING - self.report_task_info_to_driver() - - previous_components_parameters = tracker_client.get_model_run_parameters() - LOGGER.info(f"previous_components_parameters:\n{json_dumps(previous_components_parameters, indent=4)}") - - component_provider, component_parameters_on_party, user_specified_parameters = \ - ProviderManager.get_component_run_info(dsl_parser=dsl_parser, component_name=args.component_name, - role=args.role, party_id=args.party_id, - previous_components_parameters=previous_components_parameters) - RuntimeConfig.set_component_provider(component_provider) - LOGGER.info(f"component parameters on party:\n{json_dumps(component_parameters_on_party, indent=4)}") - flow_feeded_parameters = {"output_data_name": task_output_dsl.get("data")} - - # init environment, process is shared globally - RuntimeConfig.init_config(COMPUTING_ENGINE=job_parameters.computing_engine, - FEDERATION_ENGINE=job_parameters.federation_engine, - FEDERATED_MODE=job_parameters.federated_mode) - - if RuntimeConfig.COMPUTING_ENGINE == ComputingEngine.EGGROLL: - session_options = task_parameters.eggroll_run.copy() - session_options["python.path"] = os.getenv("PYTHONPATH") - session_options["python.venv"] = os.getenv("VIRTUAL_ENV") - else: - session_options = {} - - sess = session.Session(session_id=args.session_id) - sess.as_global() - sess.init_computing(computing_session_id=args.session_id, options=session_options) - component_parameters_on_party["job_parameters"] = job_parameters.to_dict() - roles = job_configuration.runtime_conf["role"] - if set(roles) == {"local"}: - LOGGER.info(f"only local roles, pass init federation") - else: - if self.is_master: - sess.init_federation(federation_session_id=args.federation_session_id, - runtime_conf=component_parameters_on_party, - service_conf=job_parameters.engines_address.get(EngineType.FEDERATION, {})) - LOGGER.info(f'run {args.component_name} {args.task_id} {args.task_version} on {args.role} {args.party_id} task') - LOGGER.info(f"component parameters on party:\n{json_dumps(component_parameters_on_party, indent=4)}") - LOGGER.info(f"task input dsl {task_input_dsl}") - task_run_args, input_table_list = self.get_task_run_args(job_id=args.job_id, role=args.role, party_id=args.party_id, - task_id=args.task_id, - task_version=args.task_version, - job_args=job_args_on_party, - job_parameters=job_parameters, - task_parameters=task_parameters, - input_dsl=task_input_dsl, - ) - if module_name in {"Upload", "Download", "Reader", "Writer", "Checkpoint"}: - task_run_args["job_parameters"] = job_parameters - # LOGGER.info(f"task input args {task_run_args}") - - need_run = component_parameters_on_party.get("ComponentParam", {}).get("need_run", True) - provider_interface = provider_utils.get_provider_interface(provider=component_provider) - run_object = provider_interface.get(module_name, ComponentRegistry.get_provider_components(provider_name=component_provider.name, provider_version=component_provider.version)).get_run_obj(self.args.role) - flow_feeded_parameters.update({"table_info": input_table_list}) - cpn_input = ComponentInput( - tracker=tracker_client, - checkpoint_manager=checkpoint_manager, - task_version_id=job_utils.generate_task_version_id(args.task_id, args.task_version), - parameters=component_parameters_on_party["ComponentParam"], - datasets=task_run_args.get("data", None), - caches=task_run_args.get("cache", None), - models=dict( - model=task_run_args.get("model"), - isometric_model=task_run_args.get("isometric_model"), - ), - job_parameters=job_parameters, - roles=dict( - role=component_parameters_on_party["role"], - local=component_parameters_on_party["local"], - ), - flow_feeded_parameters=flow_feeded_parameters, - ) - profile_log_enabled = False - try: - if int(os.getenv("FATE_PROFILE_LOG_ENABLED", "0")) > 0: - profile_log_enabled = True - except Exception as e: - LOGGER.warning(e) - if profile_log_enabled: - # add profile logs - LOGGER.info("profile logging is enabled") - profile.profile_start() - cpn_output = run_object.run(cpn_input) - sess.wait_remote_all_done() - profile.profile_ends() - else: - LOGGER.info("profile logging is disabled") - cpn_output = run_object.run(cpn_input) - sess.wait_remote_all_done() - - LOGGER.info(f"task output dsl {task_output_dsl}") - LOGGER.info(f"task output data {cpn_output.data}") - - output_table_list = [] - for index, data in enumerate(cpn_output.data): - data_name = task_output_dsl.get('data')[index] if task_output_dsl.get('data') else '{}'.format(index) - #todo: the token depends on the engine type, maybe in job parameters - persistent_table_namespace, persistent_table_name = tracker.save_output_data( - computing_table=data, - output_storage_engine=job_parameters.storage_engine, - token={"username": user_name}) - if persistent_table_namespace and persistent_table_name: - tracker.log_output_data_info(data_name=data_name, - table_namespace=persistent_table_namespace, - table_name=persistent_table_name) - output_table_list.append({"namespace": persistent_table_namespace, "name": persistent_table_name}) - self.log_output_data_table_tracker(args.job_id, input_table_list, output_table_list) - - if cpn_output.model and self.is_master: - getattr( - tracker_client if predict_tracker_client is None else predict_tracker_client, - 'save_component_output_model', - )( - model_buffers=cpn_output.model, - # There is only one model output at the current dsl version - model_alias=task_output_dsl['model'][0] if task_output_dsl.get('model') else 'default', - user_specified_run_parameters=user_specified_parameters, - ) - - if cpn_output.cache: - for i, cache in enumerate(cpn_output.cache): - if cache is None: - continue - name = task_output_dsl.get("cache")[i] if "cache" in task_output_dsl else str(i) - if isinstance(cache, DataCache): - tracker.tracking_output_cache(cache, cache_name=name) - elif isinstance(cache, tuple): - tracker.save_output_cache(cache_data=cache[0], - cache_meta=cache[1], - cache_name=name, - output_storage_engine=job_parameters.storage_engine, - output_storage_address=job_parameters.engines_address.get(EngineType.STORAGE, {}), - token={"username": user_name}) - else: - raise RuntimeError(f"can not support type {type(cache)} module run object output cache") - - self.report_info["party_status"] = TaskStatus.SUCCESS if need_run else TaskStatus.PASS - except PassError as e: - self.report_info["party_status"] = TaskStatus.PASS - except Exception as e: - traceback.print_exc() - self.report_info["party_status"] = TaskStatus.FAILED - self.generate_error_report() - LOGGER.exception(e) - try: - LOGGER.info("start destroy sessions") - sess.destroy_all_sessions() - LOGGER.info("destroy all sessions success") - except Exception as _e: - LOGGER.exception(_e) - if self.args.is_deepspeed: - raise RuntimeError(e) - finally: - try: - self.report_info["end_time"] = current_timestamp() - self.report_info["elapsed"] = self.report_info["end_time"] - start_time - self.report_task_info_to_driver() - except Exception as e: - self.report_info["party_status"] = TaskStatus.FAILED - traceback.print_exc() - LOGGER.exception(e) - msg = f"finish {args.component_name} {args.task_id} {args.task_version} on {args.role} {args.party_id} with {self.report_info['party_status']}" - LOGGER.info(msg) - print(msg) - return self.report_info - - @property - def is_master(self): - # deepspeed rank 0 - if not os.getenv("RANK"): - return True - return int(os.getenv("RANK")) == 0 - - @classmethod - def log_output_data_table_tracker(cls, job_id, input_table_list, output_table_list): - try: - parent_number = 0 - if len(input_table_list) > 1 and len(output_table_list)>1: - # TODO - return - for input_table in input_table_list: - for output_table in output_table_list: - DataTableTracker.create_table_tracker(output_table.get("name"), output_table.get("namespace"), - entity_info={ - "have_parent": True, - "parent_table_namespace": input_table.get("namespace"), - "parent_table_name": input_table.get("name"), - "parent_number": parent_number, - "job_id": job_id - }) - parent_number +=1 - except Exception as e: - LOGGER.exception(e) - - @classmethod - def get_job_args_on_party(cls, dsl_parser, job_runtime_conf, role, party_id): - party_index = job_runtime_conf["role"][role].index(int(party_id)) - job_args = dsl_parser.get_args_input() - job_args_on_party = job_args[role][party_index].get('args') if role in job_args else {} - return job_args_on_party - - @classmethod - def get_task_run_args(cls, job_id, role, party_id, task_id, task_version, - job_args, job_parameters: RunParameters, task_parameters: RunParameters, - input_dsl, filter_type=None, filter_attr=None, get_input_table=False): - task_run_args = {} - input_table = {} - input_table_info_list = [] - if 'idmapping' in role: - return {} - for input_type, input_detail in input_dsl.items(): - if filter_type and input_type not in filter_type: - continue - if input_type == 'data': - this_type_args = task_run_args[input_type] = task_run_args.get(input_type, {}) - for data_type, data_list in input_detail.items(): - data_dict = {} - for data_key in data_list: - data_key_item = data_key.split('.') - data_dict[data_key_item[0]] = {data_type: []} - for data_key in data_list: - data_key_item = data_key.split('.') - search_component_name, search_data_name = data_key_item[0], data_key_item[1] - storage_table_meta = None - tracker_client = TrackerClient(job_id=job_id, role=role, party_id=party_id, - component_name=search_component_name, - task_id=task_id, task_version=task_version) - if search_component_name == 'args': - if job_args.get('data', {}).get(search_data_name).get('namespace', '') and job_args.get( - 'data', {}).get(search_data_name).get('name', ''): - storage_table_meta = storage.StorageTableMeta( - name=job_args['data'][search_data_name]['name'], - namespace=job_args['data'][search_data_name]['namespace']) - else: - upstream_output_table_infos_json = tracker_client.get_output_data_info( - data_name=search_data_name) - if upstream_output_table_infos_json: - tracker = Tracker(job_id=job_id, role=role, party_id=party_id, - component_name=search_component_name, - task_id=task_id, task_version=task_version) - upstream_output_table_infos = [] - for _ in upstream_output_table_infos_json: - upstream_output_table_infos.append(fill_db_model_object( - Tracker.get_dynamic_db_model(TrackingOutputDataInfo, job_id)(), _)) - output_tables_meta = tracker.get_output_data_table(upstream_output_table_infos) - if output_tables_meta: - storage_table_meta = output_tables_meta.get(search_data_name, None) - args_from_component = this_type_args[search_component_name] = this_type_args.get( - search_component_name, {}) - if get_input_table and storage_table_meta: - input_table[data_key] = {'namespace': storage_table_meta.get_namespace(), - 'name': storage_table_meta.get_name()} - computing_table = None - elif storage_table_meta: - LOGGER.info(f"load computing table use {task_parameters.computing_partitions}") - computing_table = session.get_computing_session().load( - storage_table_meta.get_address(), - schema=storage_table_meta.get_schema(), - partitions=task_parameters.computing_partitions) - input_table_info_list.append({'namespace': storage_table_meta.get_namespace(), - 'name': storage_table_meta.get_name()}) - else: - computing_table = None - - if not computing_table or not filter_attr or not filter_attr.get("data", None): - data_dict[search_component_name][data_type].append(computing_table) - args_from_component[data_type] = data_dict[search_component_name][data_type] - else: - args_from_component[data_type] = dict( - [(a, getattr(computing_table, "get_{}".format(a))()) for a in filter_attr["data"]]) - elif input_type == "cache": - this_type_args = task_run_args[input_type] = task_run_args.get(input_type, {}) - for search_key in input_detail: - search_component_name, cache_name = search_key.split(".") - tracker = Tracker(job_id=job_id, role=role, party_id=party_id, component_name=search_component_name) - this_type_args[search_component_name] = tracker.get_output_cache(cache_name=cache_name) - elif input_type in {'model', 'isometric_model'}: - this_type_args = task_run_args[input_type] = task_run_args.get(input_type, {}) - for dsl_model_key in input_detail: - dsl_model_key_items = dsl_model_key.split('.') - if len(dsl_model_key_items) == 2: - search_component_name, search_model_alias = dsl_model_key_items[0], dsl_model_key_items[1] - elif len(dsl_model_key_items) == 3 and dsl_model_key_items[0] == 'pipeline': - search_component_name, search_model_alias = dsl_model_key_items[1], dsl_model_key_items[2] - else: - raise Exception('get input {} failed'.format(input_type)) - - kwargs = { - 'job_id': job_id, - 'role': role, - 'party_id': party_id, - 'component_name': search_component_name, - 'model_id': job_parameters.model_id, - # in the prediction job, job_parameters.model_version comes from the training job - 'model_version': job_parameters.model_version, - } - # get models from the training job - models = TrackerClient(**kwargs).read_component_output_model(search_model_alias) - - if not models and job_parameters.job_type == 'predict': - kwargs['model_version'] = job_id - # get models from the prediction job if not found in the training job - models = TrackerClient(**kwargs).read_component_output_model(search_model_alias) - - this_type_args[search_component_name] = models - else: - raise Exception(f"not support {input_type} input type") - if get_input_table: - return input_table - return task_run_args, input_table_info_list - - @classmethod - def monkey_patch(cls): - package_name = "monkey_patch" - package_path = os.path.join(get_fate_flow_python_directory(), "fate_flow", package_name) - if not os.path.exists(package_path): - return - for f in os.listdir(package_path): - f_path = os.path.join(get_fate_flow_python_directory(), "fate_flow", package_name, f) - if not os.path.isdir(f_path) or "__pycache__" in f_path: - continue - patch_module = importlib.import_module("fate_flow." + package_name + '.' + f + '.monkey_patch') - patch_module.patch_all() - - def generate_error_report(self): - if ERROR_REPORT: - _error = "" - etype, value, tb = sys.exc_info() - path_list = os.getenv("PYTHONPATH").split(":") - for line in traceback.TracebackException(type(value), value, tb).format(chain=True): - if not ERROR_REPORT_WITH_PATH: - for path in path_list: - line = line.replace(path, "xxx") - line = replace_ip(line) - _error += line - self.report_info["error_report"] = _error.rstrip("\n") - - -# this file may not be running on the same machine as fate_flow, -# so we need to use the tracker to get the input and save the output -if __name__ == '__main__': - worker = TaskExecutor() - worker.run() - worker.report_task_info_to_driver() - diff --git a/python/fate_flow/worker/task_initializer.py b/python/fate_flow/worker/task_initializer.py deleted file mode 100644 index 6a3a515cd..000000000 --- a/python/fate_flow/worker/task_initializer.py +++ /dev/null @@ -1,75 +0,0 @@ -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from fate_flow.utils.job_utils import get_job_configuration -from fate_flow.utils.log_utils import getLogger -from fate_flow.controller.task_controller import TaskController -from fate_flow.entity import ComponentProvider -from fate_flow.manager.provider_manager import ProviderManager -from fate_flow.utils import schedule_utils -from fate_flow.worker.base_worker import BaseWorker -from fate_flow.utils.log_utils import start_log, successful_log - -LOGGER = getLogger() - - -class TaskInitializer(BaseWorker): - def _run(self): - result = {} - job_configuration = get_job_configuration( - job_id=self.args.job_id, - role=self.args.role, - party_id=self.args.party_id - ) - dsl_parser = schedule_utils.get_job_dsl_parser( - dsl=job_configuration.dsl, - runtime_conf=job_configuration.runtime_conf, - train_runtime_conf=job_configuration.train_runtime_conf - ) - - provider = ComponentProvider(**self.args.config["provider"]) - common_task_info = self.args.config["common_task_info"] - log_msg = f"initialize the components: {self.args.config['components']}" - LOGGER.info(start_log(log_msg, role=self.args.role, party_id=self.args.party_id)) - for component_name in self.args.config["components"]: - result[component_name] = {} - task_info = {} - task_info.update(common_task_info) - - parameters, user_specified_parameters = ProviderManager.get_component_parameters(dsl_parser=dsl_parser, - component_name=component_name, - role=self.args.role, - party_id=self.args.party_id, - provider=provider) - if parameters: - task_info = {} - task_info.update(common_task_info) - task_info["component_name"] = component_name - task_info["component_module"] = parameters["module"] - task_info["provider_info"] = provider.to_dict() - task_info["component_parameters"] = parameters - TaskController.create_task(role=self.args.role, party_id=self.args.party_id, - run_on_this_party=common_task_info["run_on_this_party"], - task_info=task_info) - result[component_name]["need_run"] = True - else: - # The party does not need to run, pass - result[component_name]["need_run"] = False - LOGGER.info(successful_log(log_msg, role=self.args.role, party_id=self.args.party_id)) - return result - - -if __name__ == "__main__": - TaskInitializer().run() diff --git a/python/ofx/__init__.py b/python/ofx/__init__.py new file mode 100644 index 000000000..290575b2d --- /dev/null +++ b/python/ofx/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/python/ofx/api/__init__.py b/python/ofx/api/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/ofx/api/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/ofx/api/client.py b/python/ofx/api/client.py new file mode 100644 index 000000000..36c93ba4d --- /dev/null +++ b/python/ofx/api/client.py @@ -0,0 +1,125 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .entity import PROTOCOL + + +class BaseApi(object): + @property + def federated(self): + return + + @property + def scheduler(self): + return + + @property + def worker(self): + return + + @staticmethod + def get_name(): + return "base" + + +class FlowSchedulerApi(BaseApi): + """ + A client for communicating with a flow server. + """ + def __init__(self, host="127.0.0.1", port=9380, protocol="http", api_version=None, timeout=60, + remote_protocol="http", remote_host=None, remote_port=None, grpc_channel="default", + callback=None): + from .models.fate_flow.resource import APIClient + self.client = APIClient( + host=host, + port=port, + protocol=protocol, + api_version=api_version, + timeout=timeout, + remote_host=remote_host, + remote_port=remote_port, + remote_protocol=remote_protocol, + grpc_channel=grpc_channel + ) + self.callback = callback + + @property + def federated(self): + from .models.fate_flow.federated import Federated + return Federated(client=self.client, callback=self.callback) + + @property + def scheduler(self): + from .models.fate_flow.scheduler import Scheduler + return Scheduler(client=self.client, callback=self.callback) + + @property + def worker(self): + from .models.fate_flow.worker import Worker + return Worker(client=self.client, callback=self.callback) + + @staticmethod + def get_name(): + return PROTOCOL.FATE_FLOW + + +class BfiaSchedulerApi(BaseApi): + """ + A client for communicating with a interconnect server. + """ + def __init__(self, host="127.0.0.1", port=9380, protocol="http", api_version=None, timeout=60, + remote_protocol="http", remote_host=None, remote_port=None, grpc_channel="default", + callback=None, route_table=None, self_node_id=None): + from .models.bfia.resource import APIClient + self.client = APIClient( + host=host, + port=port, + protocol=protocol, + api_version=api_version, + timeout=timeout, + remote_host=remote_host, + remote_port=remote_port, + remote_protocol=remote_protocol, + grpc_channel=grpc_channel, + route_table=route_table, + self_node_id=self_node_id + ) + self.callback = callback + + @property + def federated(self): + from .models.bfia.federated import Federated + return Federated(client=self.client, callback=self.callback) + + @property + def scheduler(self): + from .models.bfia.scheduler import Scheduler + return Scheduler(client=self.client, callback=self.callback) + + @property + def worker(self): + from .models.bfia.worker import Worker + return Worker(client=self.client, callback=self.callback) + + @staticmethod + def get_name(): + return PROTOCOL.BFIA + + +def load_schedule_clients(**kwargs): + clients = {} + for obj in [FlowSchedulerApi]: + name = obj.get_name() + clients[name] = obj(**kwargs) + return clients diff --git a/python/ofx/api/entity.py b/python/ofx/api/entity.py new file mode 100644 index 000000000..d21cfb486 --- /dev/null +++ b/python/ofx/api/entity.py @@ -0,0 +1,53 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional, List + +import pydantic +from pydantic import BaseModel + +OSX_EXCHANGE = "" + + +class PROTOCOL: + FATE_FLOW = "fate" + BFIA = "bfia" + +class RoleSpec(BaseModel): + guest: Optional[List[str]] + host: Optional[List[str]] + arbiter: Optional[List[str]] + local: Optional[List[str]] + + +class BFIAHttpHeadersSpec(pydantic.BaseModel): + x_auth_sign: Optional[str] + x_node_id: Optional[str] + x_nonce: Optional[str] + x_trace_id: Optional[str] + x_timestamp: Optional[str] + + +class BFIAHeadersSpec(pydantic.BaseModel): + x_ptp_version: Optional[str] + x_ptp_provider_code: Optional[str] + x_ptp_trace_id: Optional[str] + x_ptp_token: Optional[str] + x_ptp_uri: Optional[str] + x_ptp_from_node_id: Optional[str] + x_ptp_from_inst_id: Optional[str] + x_ptp_target_node_id: Optional[str] + x_ptp_target_inst_id: Optional[str] + x_ptp_session_id: Optional[str] + x_ptp_topic: Optional[str] diff --git a/python/ofx/api/models/__init__.py b/python/ofx/api/models/__init__.py new file mode 100644 index 000000000..0e6a37ecc --- /dev/null +++ b/python/ofx/api/models/__init__.py @@ -0,0 +1,17 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ofx.api.models import bfia, fate_flow + +__all__ = [bfia, fate_flow] diff --git a/python/ofx/api/models/bfia/__init__.py b/python/ofx/api/models/bfia/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/ofx/api/models/bfia/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/ofx/api/models/bfia/federated.py b/python/ofx/api/models/bfia/federated.py new file mode 100644 index 000000000..bb2bb572f --- /dev/null +++ b/python/ofx/api/models/bfia/federated.py @@ -0,0 +1,57 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .resource import BaseAPI + + +class Federated(BaseAPI): + def create_job(self, node_list, command_body): + return self.job_command( + node_list=node_list, + command_body=command_body, + parallel=False, + endpoint="/v1/interconn/schedule/job/create" + ) + + def start_job(self, node_list, command_body): + return self.job_command( + node_list=node_list, + command_body=command_body, + parallel=False, + endpoint="/v1/interconn/schedule/job/start" + ) + + def stop_job(self, node_list, command_body): + return self.job_command( + node_list=node_list, + command_body=command_body, + parallel=False, + endpoint="/v1/interconn/schedule/job/stop" + ) + + def start_task(self, node_list, command_body): + return self.job_command( + node_list=node_list, + command_body=command_body, + parallel=False, + endpoint="/v1/interconn/schedule/task/start" + ) + + def poll_task(self, node_list, command_body): + return self.job_command( + node_list=node_list, + command_body=command_body, + parallel=False, + endpoint="/v1/interconn/schedule/task/poll" + ) diff --git a/python/ofx/api/models/bfia/resource.py b/python/ofx/api/models/bfia/resource.py new file mode 100644 index 000000000..df8641788 --- /dev/null +++ b/python/ofx/api/models/bfia/resource.py @@ -0,0 +1,233 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import threading +import time +import uuid + +import requests + +from ...entity import BFIAHttpHeadersSpec + +FEDERATED_ERROR = 104 + + +class APIClient(requests.Session): + def __init__(self, host="127.0.0.1", port=9380, protocol="http", api_version=None, timeout=60, + remote_protocol="http", remote_host=None, remote_port=None, grpc_channel="default", + provider="FATE", route_table=None, self_node_id=""): + super().__init__() + self.host = host + self.port = port + self.protocol = protocol + self.timeout = timeout + self.api_version = api_version + self.remote_protocol = remote_protocol + self.remote_host = remote_host + self.remote_port = remote_port + self.grpc_channel = grpc_channel + self.provider = provider + self.route_table = route_table + self.node_id = self_node_id + + @property + def base_url(self): + return f'{self.protocol}://{self.host}:{self.port}' + + @property + def version(self): + if self.api_version: + return self.api_version + return None + + def post(self, endpoint, data=None, json=None, **kwargs): + return self.request('POST', url=self._set_url(endpoint), data=data, json=json, + **self._set_request_timeout(kwargs)) + + def send_file(self, endpoint, data=None, json=None, params=None, files=None, **kwargs): + return self.request('POST', url=self._set_url(endpoint), data=data, json=json, files=files, params=params, + **self._set_request_timeout(kwargs)) + + def get(self, endpoint, **kwargs): + kwargs.setdefault('allow_redirects', True) + return self.request('GET', url=self._set_url(endpoint), **self._set_request_timeout(kwargs)) + + def put(self, endpoint, data=None, **kwargs): + return self.request('PUT', url=self._set_url(endpoint), data=data, **self._set_request_timeout(kwargs)) + + def delete(self, endpoint, **kwargs): + return self.request('DELETE', url=self._set_url(endpoint), **self._set_request_timeout(kwargs)) + + @property + def url(self): + return self._url + + @property + def _url(self): + if self.version: + return f"{self.base_url}/{self.version}" + else: + return self.base_url + + def generate_endpoint(self, endpoint): + if self.version: + return f"{endpoint}/{self.version}" + else: + return endpoint + + def _set_request_timeout(self, kwargs): + kwargs.setdefault('timeout', self.timeout) + return kwargs + + def _set_url(self, endpoint): + return f"{self._url}/{endpoint}" + + def remote( + self, method, endpoint, dest_node_id, body, is_local=False, extra_params=None, + headers=None + ): + + if not self.route_table: + raise Exception(f'Route table is null') + if not headers: + headers = {} + headers.update( + BFIAHttpHeadersSpec( + x_auth_sign="", + x_node_id=self.node_id, + x_nonce=str(uuid.uuid4()), + x_trace_id="", + x_timestamp=str(int(time.time() * 1000)) + ).dict() + ) + kwargs = { + 'method': method, + 'endpoint': endpoint, + 'json_body': body, + "headers": headers + } + + if extra_params: + kwargs.update(extra_params) + + if is_local: + return self.remote_on_http(**kwargs) + + elif dest_node_id in self.route_table: + kwargs.update({ + "host": self.route_table[dest_node_id]["host"], + "port": self.route_table[dest_node_id]["port"], + }) + return self.remote_on_http(**kwargs) + + else: + raise Exception(f'No found node id {dest_node_id} in route table: {self.route_table}') + + def remote_on_http(self, method, endpoint, host=None, port=None, try_times=3, timeout=10, json_body=None, + headers=None, **kwargs): + if host and port: + url = f"http://{host}:{port}{endpoint}" + else: + url = f"{self.base_url}{endpoint}" + for t in range(try_times): + try: + response = requests.request(method=method, url=url, timeout=timeout, json=json_body, headers=headers) + response.raise_for_status() + except Exception as e: + if t >= try_times - 1: + raise e + else: + try: + return response.json() + except: + raise Exception(response.text) + + +class BaseAPI: + def __init__(self, client: APIClient, callback=None): + self.client = client + self.callback = callback + + def federated_command( + self, dest_node_id, endpoint, body, federated_response, method='POST', only_scheduler=False, + extra_params=None + ): + try: + headers = {} + response = self.client.remote( + method=method, + endpoint=endpoint, + dest_node_id=dest_node_id, + body=body if body else {}, + extra_params=extra_params, + is_local=self.is_local(node_id=dest_node_id), + headers=headers + ) + if only_scheduler: + return response + except Exception as e: + response = { + "code": FEDERATED_ERROR, + "message": "Federated schedule error, {}".format(e) + } + if only_scheduler: + return response + federated_response[dest_node_id] = response + + @staticmethod + def is_local(node_id): + return node_id == "0" + + def job_command(self, node_list, endpoint, command_body=None, parallel=False, method="POST"): + federated_response = {} + threads = [] + if not command_body: + command_body = {} + for node_id in node_list: + federated_response[node_id] = {} + kwargs = { + "dest_node_id": node_id, + "endpoint": endpoint, + "body": command_body, + "federated_response": federated_response, + "method": method + } + if parallel: + t = threading.Thread(target=self.federated_command, kwargs=kwargs) + threads.append(t) + t.start() + else: + self.federated_command(**kwargs) + for thread in threads: + thread.join() + return federated_response + + + def scheduler_command(self, endpoint, node_id, command_body=None, method='POST'): + try: + federated_response = {} + response = self.federated_command( + method=method, + endpoint=endpoint, + dest_node_id=node_id, + body=command_body if command_body else {}, + federated_response=federated_response, + only_scheduler=True, + ) + except Exception as e: + response = { + "code": FEDERATED_ERROR, + "message": "Federated schedule error, {}".format(e) + } + return response diff --git a/python/ofx/api/models/bfia/scheduler.py b/python/ofx/api/models/bfia/scheduler.py new file mode 100644 index 000000000..5a41ddabc --- /dev/null +++ b/python/ofx/api/models/bfia/scheduler.py @@ -0,0 +1,49 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .resource import BaseAPI + + +class Scheduler(BaseAPI): + def create_job(self, party_id, command_body): + return self.scheduler_command( + endpoint="/v1/interconn/schedule/job/create_all", + node_id=party_id, + command_body=command_body + ) + + def audit_confirm(self, party_id, command_body): + return self.scheduler_command( + endpoint="/v1/interconn/schedule/job/audit_confirm", + node_id=party_id, + command_body=command_body + ) + + def stop_job(self, party_id, command_body): + return self.scheduler_command( + endpoint="/v1/interconn/schedule/job/stop_all", + node_id=party_id, + command_body=command_body + ) + + def report_task(self, party_id, command_body): + return self.scheduler_command( + endpoint="/v1/interconn/schedule/task/callback", + node_id=party_id, + command_body=command_body + ) + + + + diff --git a/python/ofx/api/models/bfia/worker.py b/python/ofx/api/models/bfia/worker.py new file mode 100644 index 000000000..66a98e0bb --- /dev/null +++ b/python/ofx/api/models/bfia/worker.py @@ -0,0 +1,26 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .resource import BaseAPI + + +class Worker(BaseAPI): + def report_task_status(self, status, task_id, role): + return self.client.post( + endpoint="/v1/platform/schedule/task/callback", + json={ + "status": status, + "task_id": task_id, + "role": role + }) diff --git a/python/ofx/api/models/fate_flow/__init__.py b/python/ofx/api/models/fate_flow/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/ofx/api/models/fate_flow/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/ofx/api/models/fate_flow/federated.py b/python/ofx/api/models/fate_flow/federated.py new file mode 100644 index 000000000..d5f5dc356 --- /dev/null +++ b/python/ofx/api/models/fate_flow/federated.py @@ -0,0 +1,66 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .resource import BaseAPI + + +class Federated(BaseAPI): + def create_job(self, job_id, roles, initiator_party_id, command_body): + return self.job_command(job_id=job_id, roles=roles, command="create", command_body=command_body, + initiator_party_id=initiator_party_id, parallel=False) + + def stop_job(self, job_id, roles): + return self.job_command(job_id=job_id, roles=roles, command="stop") + + def sync_job_status(self, job_id, roles, command_body=None): + return self.job_command(job_id=job_id, roles=roles, command=f"status/update", command_body=command_body) + + def resource_for_job(self, job_id, roles, operation_type): + return self.job_command(job_id=job_id, roles=roles, command=f"resource/{operation_type}") + + def start_job(self, job_id, roles, command_body=None): + return self.job_command(job_id=job_id, roles=roles, command="start", command_body=command_body) + + def update_job(self, job_id, roles, command_body=None): + return self.job_command(job_id=job_id, roles=roles, command="update", command_body=command_body) + + def save_pipelined_model(self, job_id, roles): + return self.job_command(job_id=job_id, roles=roles, command="pipeline/save") + + def clean_job(self, job_id, roles, command_body=None): + return self.job_command(job_id=job_id, roles=roles, command="clean", command_body=command_body) + + def resource_for_task(self, tasks, operation_type): + return self.task_command(tasks=tasks, command=f"resource/{operation_type}") + + def create_task(self, tasks, command_body=None): + return self.task_command(tasks=tasks, command="create", command_body=command_body) + + def start_task(self, tasks): + return self.task_command(tasks=tasks, command="start") + + def rerun_task(self, tasks, task_version): + return self.task_command(tasks=tasks, command="rerun", command_body={"new_version": task_version}) + + def collect_task(self, tasks): + return self.task_command(tasks=tasks, command="collect") + + def sync_task_status(self, tasks, command_body=None): + return self.task_command(tasks=tasks, command=f"status/update", command_body=command_body) + + def stop_task(self, tasks, command_body=None): + return self.task_command(tasks=tasks, command="stop", command_body=command_body) + + def clean_task(self, tasks, content_type): + return self.task_command(tasks=tasks, command="clean/{}".format(content_type)) diff --git a/python/ofx/api/models/fate_flow/resource.py b/python/ofx/api/models/fate_flow/resource.py new file mode 100644 index 000000000..de4c3a274 --- /dev/null +++ b/python/ofx/api/models/fate_flow/resource.py @@ -0,0 +1,309 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import threading + +import json +import requests + +from ...utils.grpc_utils import wrap_proxy_grpc_packet +from ...utils.grpc_utils import gen_routing_metadata, get_proxy_channel + +FEDERATED_ERROR = 104 + + +class APIClient(requests.Session): + def __init__(self, host="127.0.0.1", port=9380, protocol="http", api_version=None, timeout=60, + remote_protocol="http", remote_host=None, remote_port=None, grpc_channel="default", + provider: str = "FATE", route_table=None): + super().__init__() + self.host = host + self.port = port + self.protocol = protocol + self.timeout = timeout + self.api_version = api_version + self.remote_protocol = remote_protocol + self.remote_host = remote_host + self.remote_port = remote_port + self.grpc_channel = grpc_channel + self.provider = provider + self.route_table = route_table + + @property + def base_url(self): + return f'{self.protocol}://{self.host}:{self.port}' + + @property + def version(self): + if self.api_version: + return self.api_version + return None + + def post(self, endpoint, data=None, json=None, **kwargs): + return self.request('POST', url=self._set_url(endpoint), data=data, json=json, + **self._set_request_timeout(kwargs)) + + def send_file(self, endpoint, data=None, json=None, params=None, files=None, **kwargs): + return self.request('POST', url=self._set_url(endpoint), data=data, json=json, files=files, params=params, + **self._set_request_timeout(kwargs)) + + def get(self, endpoint, **kwargs): + kwargs.setdefault('allow_redirects', True) + return self.request('GET', url=self._set_url(endpoint), **self._set_request_timeout(kwargs)) + + def put(self, endpoint, data=None, **kwargs): + return self.request('PUT', url=self._set_url(endpoint), data=data, **self._set_request_timeout(kwargs)) + + def delete(self, endpoint, **kwargs): + return self.request('DELETE', url=self._set_url(endpoint), **self._set_request_timeout(kwargs)) + + @property + def url(self): + return self._url + + @property + def _url(self): + if self.version: + return f"{self.base_url}/{self.version}" + else: + return self.base_url + + def generate_endpoint(self, endpoint): + if self.version: + return f"{endpoint}/{self.version}" + else: + return endpoint + + def _set_request_timeout(self, kwargs): + kwargs.setdefault('timeout', self.timeout) + return kwargs + + def _set_url(self, endpoint): + return f"{self._url}/{endpoint}" + + def remote(self, job_id, method, endpoint, src_party_id, dest_party_id, src_role, json_body, is_local=False, + extra_params=None, headers=None): + if not headers: + headers = {} + if self.version: + endpoint = f"/{self.version}{endpoint}" + kwargs = { + 'job_id': job_id, + 'method': method, + 'endpoint': endpoint, + 'src_party_id': src_party_id, + 'dest_party_id': dest_party_id, + 'src_role': src_role, + 'json_body': json_body, + "headers": headers + } + if extra_params: + kwargs.update(extra_params) + if not self.remote_host and not self.remote_port and self.remote_protocol == "grpc": + raise Exception( + f'{self.remote_protocol} coordination communication protocol need remote host and remote port.') + kwargs.update({ + "source_host": self.host, + "source_port": self.port, + }) + if is_local: + return self.remote_on_http(**kwargs) + if self.remote_host and self.remote_port: + kwargs.update({ + "host": self.remote_host, + "port": self.remote_port, + }) + if self.remote_protocol == "http": + return self.remote_on_http(**kwargs) + if self.remote_protocol == "grpc": + return self.remote_on_grpc_proxy(**kwargs) + else: + raise Exception(f'{self.remote_protocol} coordination communication protocol is not supported.') + else: + return self.remote_on_http(**kwargs) + + def remote_on_http(self, method, endpoint, host=None, port=None, try_times=3, timeout=10, + json_body=None, dest_party_id=None, service_name="fateflow", headers=None, **kwargs): + headers.update({ + "dest-party-id": dest_party_id, + "service": service_name + }) + if host and port: + url = f"{self.remote_protocol}://{host}:{port}{endpoint}" + else: + url = f"{self.base_url}{endpoint}" + for t in range(try_times): + try: + response = requests.request(method=method, url=url, timeout=timeout, json=json_body, headers=headers) + response.raise_for_status() + except Exception as e: + if t >= try_times - 1: + raise e + else: + try: + return response.json() + except: + raise Exception(response.text) + + @staticmethod + def remote_on_grpc_proxy(job_id, method, host, port, endpoint, src_party_id, dest_party_id, json_body, + try_times=3, timeout=10, headers=None, source_host=None, source_port=None, **kwargs): + _packet = wrap_proxy_grpc_packet( + json_body=json_body, http_method=method, url=endpoint, + src_party_id=src_party_id, dst_party_id=dest_party_id, + job_id=job_id, headers=headers, overall_timeout=timeout, + source_host=source_host, source_port=source_port + ) + _routing_metadata = gen_routing_metadata( + src_party_id=src_party_id, dest_party_id=dest_party_id, + ) + for t in range(try_times): + channel, stub = get_proxy_channel(host, port) + + try: + _return, _call = stub.unaryCall.with_call( + _packet, metadata=_routing_metadata, + timeout=timeout or None, + ) + except Exception as e: + if t >= try_times - 1: + raise e + else: + try: + return json.loads(bytes.decode(_return.body.value)) + except Exception: + raise RuntimeError(f"{_return}, {_call}") + finally: + channel.close() + + +class BaseAPI: + def __init__(self, client: APIClient, callback=None): + self.client = client + self.callback = callback + + def federated_command(self, job_id, src_role, src_party_id, dest_role, dest_party_id, endpoint, body, + federated_response, method='POST', only_scheduler=False, extra_params=None, + initiator_party_id=""): + try: + headers = {} + if self.callback: + result = self.callback(dest_party_id, body, initiator_party_id=initiator_party_id) + if result.code == 0: + headers = result.signature if result.signature else {} + else: + raise Exception(result.code, result.message) + headers.update({"initiator-party-id": initiator_party_id}) + response = self.client.remote(job_id=job_id, + method=method, + endpoint=endpoint, + src_role=src_role, + src_party_id=src_party_id, + dest_party_id=dest_party_id, + json_body=body if body else {}, + extra_params=extra_params, + is_local=self.is_local(party_id=dest_party_id), + headers=headers) + if only_scheduler: + return response + except Exception as e: + response = { + "code": FEDERATED_ERROR, + "message": "Federated schedule error, {}".format(e) + } + if only_scheduler: + return response + federated_response[dest_role][dest_party_id] = response + + @staticmethod + def is_local(party_id): + return party_id == "0" + + def job_command(self, job_id, roles, command, command_body=None, parallel=False, initiator_party_id=""): + federated_response = {} + api_type = "partner/job" + threads = [] + if not command_body: + command_body = {} + for party in roles: + dest_role = party.get("role") + dest_party_ids = party.get("party_id") + federated_response[dest_role] = {} + for dest_party_id in dest_party_ids: + endpoint = f"/{api_type}/{command}" + command_body["role"] = dest_role + command_body["party_id"] = dest_party_id + command_body["job_id"] = job_id + args = (job_id, "", "", dest_role, dest_party_id, endpoint, command_body, federated_response) + kwargs = {"initiator_party_id": initiator_party_id} + if parallel: + t = threading.Thread(target=self.federated_command, args=args, kwargs=kwargs) + threads.append(t) + t.start() + else: + self.federated_command(*args, initiator_party_id=initiator_party_id) + for thread in threads: + thread.join() + return federated_response + + def task_command(self, tasks, command, command_body=None, parallel=False): + federated_response = {} + threads = [] + if not command_body: + command_body = {} + for task in tasks: + command_body.update({ + "job_id": task["job_id"], + "role": task["role"], + "party_id": task["party_id"], + "task_id": task["task_id"], + "task_version": task["task_version"] + }) + dest_role, dest_party_id = task["role"], task["party_id"] + federated_response[dest_role] = federated_response.get(dest_role, {}) + endpoint = f"/partner/task/{command}" + args = (task['job_id'], task['role'], task['party_id'], dest_role, dest_party_id, endpoint, command_body, + federated_response) + if parallel: + t = threading.Thread(target=self.federated_command, args=args) + threads.append(t) + t.start() + else: + self.federated_command(*args) + for thread in threads: + thread.join() + return federated_response + + def scheduler_command(self, command, party_id, command_body=None, method='POST', initiator_party_id=""): + try: + federated_response = {} + endpoint = f"/scheduler/{command}" + response = self.federated_command(job_id="", + method=method, + endpoint=endpoint, + src_role="", + src_party_id="", + dest_role="", + dest_party_id=party_id, + body=command_body if command_body else {}, + federated_response=federated_response, + only_scheduler=True, + initiator_party_id=initiator_party_id + ) + except Exception as e: + response = { + "code": FEDERATED_ERROR, + "message": "Federated schedule error, {}".format(e) + } + return response diff --git a/python/ofx/api/models/fate_flow/scheduler.py b/python/ofx/api/models/fate_flow/scheduler.py new file mode 100644 index 000000000..0547c98b1 --- /dev/null +++ b/python/ofx/api/models/fate_flow/scheduler.py @@ -0,0 +1,42 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .resource import BaseAPI + + +class Scheduler(BaseAPI): + def create_job(self, party_id, initiator_party_id, command_body): + return self.scheduler_command(command="job/create", + party_id=party_id, + initiator_party_id=initiator_party_id, + command_body=command_body + ) + + def stop_job(self, party_id, command_body): + return self.scheduler_command(command="job/stop", + party_id=party_id, + command_body=command_body + ) + + def rerun_job(self, party_id, command_body): + return self.scheduler_command(command="job/rerun", + party_id=party_id, + command_body=command_body + ) + + def report_task(self, party_id, command_body): + return self.scheduler_command(command="task/report", + party_id=party_id, + command_body=command_body + ) diff --git a/python/ofx/api/models/fate_flow/worker.py b/python/ofx/api/models/fate_flow/worker.py new file mode 100644 index 000000000..fbd380686 --- /dev/null +++ b/python/ofx/api/models/fate_flow/worker.py @@ -0,0 +1,110 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .resource import BaseAPI + + +class Worker(BaseAPI): + def report_task_status(self, status, execution_id, error=""): + if not error: + error = "" + endpoint = '/worker/task/status' + return self.client.post(endpoint=endpoint, json={ + "status": status, + "execution_id": execution_id, + "error": error + }) + + def query_task_status(self, execution_id): + endpoint = '/worker/task/status' + return self.client.get(endpoint=endpoint, json={ + "execution_id": execution_id, + }) + + def save_model(self, model_id, model_version, execution_id, output_key, type_name, fp): + files = {"file": fp} + return self.client.send_file( + endpoint="/worker/model/save", + files=files, + data={ + "model_id": model_id, + "model_version": model_version, + "execution_id": execution_id, + "output_key": output_key, + "type_name": type_name + }) + + def save_data_tracking(self, execution_id, output_key, meta_data, uri, namespace, name, overview, source, data_type, + index, partitions=None): + return self.client.post( + endpoint="/worker/data/tracking/save", + json={ + "execution_id": execution_id, + "output_key": output_key, + "meta_data": meta_data, + "uri": uri, + "namespace": namespace, + "name": name, + "overview": overview, + "source": source, + "data_type": data_type, + "index": index, + "partitions": partitions + }) + + def query_data_meta(self, job_id=None, role=None, party_id=None, task_name=None, output_key=None, namespace=None, + name=None): + # [job_id, role, party_id, task_name, output_key] or [name, namespace] + if namespace and name: + params = { + "namespace": namespace, + "name": name + } + else: + params = { + "job_id": job_id, + "role": role, + "party_id": party_id, + "task_name": task_name, + "output_key": output_key + } + return self.client.get( + endpoint="/worker/data/tracking/query", + params=params + ) + + def download_model(self, model_id, model_version, task_name, output_key, role, party_id): + return self.client.get( + endpoint="/worker/model/download", + params={ + "model_id": model_id, + "model_version": model_version, + "task_name": task_name, + "output_key": output_key, + "role": role, + "party_id": party_id + } + ) + + def save_metric(self, execution_id, data): + return self.client.post( + endpoint="/worker/metric/save", + json={ + "execution_id": execution_id, + "data": data + }) + + def get_metric_save_url(self, execution_id): + endpoint = f"/worker/metric/save/{execution_id}" + return f"{self.client.url}{endpoint}" diff --git a/python/ofx/api/proto/__init__.py b/python/ofx/api/proto/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/ofx/api/proto/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/ofx/api/proto/rollsite/__init__.py b/python/ofx/api/proto/rollsite/__init__.py new file mode 100644 index 000000000..f958736b0 --- /dev/null +++ b/python/ofx/api/proto/rollsite/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys + +_pb_path = os.path.abspath(os.path.join(__file__, os.path.pardir)) +if _pb_path not in sys.path: + sys.path.append(_pb_path) \ No newline at end of file diff --git a/python/ofx/api/proto/rollsite/basic_meta_pb2.py b/python/ofx/api/proto/rollsite/basic_meta_pb2.py new file mode 100644 index 000000000..ca81e5d1f --- /dev/null +++ b/python/ofx/api/proto/rollsite/basic_meta_pb2.py @@ -0,0 +1,209 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: basic-meta.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x10\x62\x61sic-meta.proto\x12\x1e\x63om.webank.ai.eggroll.api.core"6\n\x08\x45ndpoint\x12\n\n\x02ip\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\x12\x10\n\x08hostname\x18\x03 \x01(\t"H\n\tEndpoints\x12;\n\tendpoints\x18\x01 \x03(\x0b\x32(.com.webank.ai.eggroll.api.core.Endpoint"H\n\x04\x44\x61ta\x12\x0e\n\x06isNull\x18\x01 \x01(\x08\x12\x14\n\x0chostLanguage\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c"F\n\x0cRepeatedData\x12\x36\n\x08\x64\x61talist\x18\x01 \x03(\x0b\x32$.com.webank.ai.eggroll.api.core.Data"u\n\x0b\x43\x61llRequest\x12\x0f\n\x07isAsync\x18\x01 \x01(\x08\x12\x0f\n\x07timeout\x18\x02 \x01(\x03\x12\x0f\n\x07\x63ommand\x18\x03 \x01(\t\x12\x33\n\x05param\x18\x04 \x01(\x0b\x32$.com.webank.ai.eggroll.api.core.Data"\x88\x01\n\x0c\x43\x61llResponse\x12\x42\n\x0creturnStatus\x18\x01 \x01(\x0b\x32,.com.webank.ai.eggroll.api.core.ReturnStatus\x12\x34\n\x06result\x18\x02 \x01(\x0b\x32$.com.webank.ai.eggroll.api.core.Data""\n\x03Job\x12\r\n\x05jobId\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t"Y\n\x04Task\x12\x30\n\x03job\x18\x01 \x01(\x0b\x32#.com.webank.ai.eggroll.api.core.Job\x12\x0e\n\x06taskId\x18\x02 \x01(\x03\x12\x0f\n\x07tableId\x18\x03 \x01(\x03"N\n\x06Result\x12\x32\n\x04task\x18\x01 \x01(\x0b\x32$.com.webank.ai.eggroll.api.core.Task\x12\x10\n\x08resultId\x18\x02 \x01(\x03"-\n\x0cReturnStatus\x12\x0c\n\x04\x63ode\x18\x01 \x01(\x05\x12\x0f\n\x07message\x18\x02 \x01(\t"\xe2\x01\n\x0bSessionInfo\x12\x11\n\tsessionId\x18\x01 \x01(\t\x12\x61\n\x13\x63omputingEngineConf\x18\x02 \x03(\x0b\x32\x44.com.webank.ai.eggroll.api.core.SessionInfo.ComputingEngineConfEntry\x12\x14\n\x0cnamingPolicy\x18\x03 \x01(\t\x12\x0b\n\x03tag\x18\x04 \x01(\t\x1a:\n\x18\x43omputingEngineConfEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x62\x06proto3' +) + + +_ENDPOINT = DESCRIPTOR.message_types_by_name["Endpoint"] +_ENDPOINTS = DESCRIPTOR.message_types_by_name["Endpoints"] +_DATA = DESCRIPTOR.message_types_by_name["Data"] +_REPEATEDDATA = DESCRIPTOR.message_types_by_name["RepeatedData"] +_CALLREQUEST = DESCRIPTOR.message_types_by_name["CallRequest"] +_CALLRESPONSE = DESCRIPTOR.message_types_by_name["CallResponse"] +_JOB = DESCRIPTOR.message_types_by_name["Job"] +_TASK = DESCRIPTOR.message_types_by_name["Task"] +_RESULT = DESCRIPTOR.message_types_by_name["Result"] +_RETURNSTATUS = DESCRIPTOR.message_types_by_name["ReturnStatus"] +_SESSIONINFO = DESCRIPTOR.message_types_by_name["SessionInfo"] +_SESSIONINFO_COMPUTINGENGINECONFENTRY = _SESSIONINFO.nested_types_by_name[ + "ComputingEngineConfEntry" +] +Endpoint = _reflection.GeneratedProtocolMessageType( + "Endpoint", + (_message.Message,), + { + "DESCRIPTOR": _ENDPOINT, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Endpoint) + }, +) +_sym_db.RegisterMessage(Endpoint) + +Endpoints = _reflection.GeneratedProtocolMessageType( + "Endpoints", + (_message.Message,), + { + "DESCRIPTOR": _ENDPOINTS, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Endpoints) + }, +) +_sym_db.RegisterMessage(Endpoints) + +Data = _reflection.GeneratedProtocolMessageType( + "Data", + (_message.Message,), + { + "DESCRIPTOR": _DATA, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Data) + }, +) +_sym_db.RegisterMessage(Data) + +RepeatedData = _reflection.GeneratedProtocolMessageType( + "RepeatedData", + (_message.Message,), + { + "DESCRIPTOR": _REPEATEDDATA, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.RepeatedData) + }, +) +_sym_db.RegisterMessage(RepeatedData) + +CallRequest = _reflection.GeneratedProtocolMessageType( + "CallRequest", + (_message.Message,), + { + "DESCRIPTOR": _CALLREQUEST, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.CallRequest) + }, +) +_sym_db.RegisterMessage(CallRequest) + +CallResponse = _reflection.GeneratedProtocolMessageType( + "CallResponse", + (_message.Message,), + { + "DESCRIPTOR": _CALLRESPONSE, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.CallResponse) + }, +) +_sym_db.RegisterMessage(CallResponse) + +Job = _reflection.GeneratedProtocolMessageType( + "Job", + (_message.Message,), + { + "DESCRIPTOR": _JOB, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Job) + }, +) +_sym_db.RegisterMessage(Job) + +Task = _reflection.GeneratedProtocolMessageType( + "Task", + (_message.Message,), + { + "DESCRIPTOR": _TASK, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Task) + }, +) +_sym_db.RegisterMessage(Task) + +Result = _reflection.GeneratedProtocolMessageType( + "Result", + (_message.Message,), + { + "DESCRIPTOR": _RESULT, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Result) + }, +) +_sym_db.RegisterMessage(Result) + +ReturnStatus = _reflection.GeneratedProtocolMessageType( + "ReturnStatus", + (_message.Message,), + { + "DESCRIPTOR": _RETURNSTATUS, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.ReturnStatus) + }, +) +_sym_db.RegisterMessage(ReturnStatus) + +SessionInfo = _reflection.GeneratedProtocolMessageType( + "SessionInfo", + (_message.Message,), + { + "ComputingEngineConfEntry": _reflection.GeneratedProtocolMessageType( + "ComputingEngineConfEntry", + (_message.Message,), + { + "DESCRIPTOR": _SESSIONINFO_COMPUTINGENGINECONFENTRY, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.SessionInfo.ComputingEngineConfEntry) + }, + ), + "DESCRIPTOR": _SESSIONINFO, + "__module__": "basic_meta_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.SessionInfo) + }, +) +_sym_db.RegisterMessage(SessionInfo) +_sym_db.RegisterMessage(SessionInfo.ComputingEngineConfEntry) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _SESSIONINFO_COMPUTINGENGINECONFENTRY._options = None + _SESSIONINFO_COMPUTINGENGINECONFENTRY._serialized_options = b"8\001" + _ENDPOINT._serialized_start = 52 + _ENDPOINT._serialized_end = 106 + _ENDPOINTS._serialized_start = 108 + _ENDPOINTS._serialized_end = 180 + _DATA._serialized_start = 182 + _DATA._serialized_end = 254 + _REPEATEDDATA._serialized_start = 256 + _REPEATEDDATA._serialized_end = 326 + _CALLREQUEST._serialized_start = 328 + _CALLREQUEST._serialized_end = 445 + _CALLRESPONSE._serialized_start = 448 + _CALLRESPONSE._serialized_end = 584 + _JOB._serialized_start = 586 + _JOB._serialized_end = 620 + _TASK._serialized_start = 622 + _TASK._serialized_end = 711 + _RESULT._serialized_start = 713 + _RESULT._serialized_end = 791 + _RETURNSTATUS._serialized_start = 793 + _RETURNSTATUS._serialized_end = 838 + _SESSIONINFO._serialized_start = 841 + _SESSIONINFO._serialized_end = 1067 + _SESSIONINFO_COMPUTINGENGINECONFENTRY._serialized_start = 1009 + _SESSIONINFO_COMPUTINGENGINECONFENTRY._serialized_end = 1067 +# @@protoc_insertion_point(module_scope) diff --git a/python/ofx/api/proto/rollsite/proxy_pb2.py b/python/ofx/api/proto/rollsite/proxy_pb2.py new file mode 100644 index 000000000..3bbd35d6a --- /dev/null +++ b/python/ofx/api/proto/rollsite/proxy_pb2.py @@ -0,0 +1,197 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: proxy.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import enum_type_wrapper + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x0bproxy.proto\x12*com.webank.ai.eggroll.api.networking.proxy\x1a\x10\x62\x61sic-meta.proto"&\n\x05Model\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07\x64\x61taKey\x18\x02 \x01(\t"X\n\x04Task\x12\x0e\n\x06taskId\x18\x01 \x01(\t\x12@\n\x05model\x18\x02 \x01(\x0b\x32\x31.com.webank.ai.eggroll.api.networking.proxy.Model"p\n\x05Topic\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07partyId\x18\x02 \x01(\t\x12\x0c\n\x04role\x18\x03 \x01(\t\x12:\n\x08\x63\x61llback\x18\x04 \x01(\x0b\x32(.com.webank.ai.eggroll.api.core.Endpoint"\x17\n\x07\x43ommand\x12\x0c\n\x04name\x18\x01 \x01(\t"p\n\x04\x43onf\x12\x16\n\x0eoverallTimeout\x18\x01 \x01(\x03\x12\x1d\n\x15\x63ompletionWaitTimeout\x18\x02 \x01(\x03\x12\x1d\n\x15packetIntervalTimeout\x18\x03 \x01(\x03\x12\x12\n\nmaxRetries\x18\x04 \x01(\x05"\x9a\x03\n\x08Metadata\x12>\n\x04task\x18\x01 \x01(\x0b\x32\x30.com.webank.ai.eggroll.api.networking.proxy.Task\x12>\n\x03src\x18\x02 \x01(\x0b\x32\x31.com.webank.ai.eggroll.api.networking.proxy.Topic\x12>\n\x03\x64st\x18\x03 \x01(\x0b\x32\x31.com.webank.ai.eggroll.api.networking.proxy.Topic\x12\x44\n\x07\x63ommand\x18\x04 \x01(\x0b\x32\x33.com.webank.ai.eggroll.api.networking.proxy.Command\x12\x10\n\x08operator\x18\x05 \x01(\t\x12\x0b\n\x03seq\x18\x06 \x01(\x03\x12\x0b\n\x03\x61\x63k\x18\x07 \x01(\x03\x12>\n\x04\x63onf\x18\x08 \x01(\x0b\x32\x30.com.webank.ai.eggroll.api.networking.proxy.Conf\x12\x0b\n\x03\x65xt\x18\t \x01(\x0c\x12\x0f\n\x07version\x18\x64 \x01(\t""\n\x04\x44\x61ta\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c"\x8e\x01\n\x06Packet\x12\x44\n\x06header\x18\x01 \x01(\x0b\x32\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata\x12>\n\x04\x62ody\x18\x02 \x01(\x0b\x32\x30.com.webank.ai.eggroll.api.networking.proxy.Data"\xa3\x01\n\x11HeartbeatResponse\x12\x44\n\x06header\x18\x01 \x01(\x0b\x32\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata\x12H\n\toperation\x18\x02 \x01(\x0e\x32\x35.com.webank.ai.eggroll.api.networking.proxy.Operation"\xc5\x01\n\x0cPollingFrame\x12\x0e\n\x06method\x18\x01 \x01(\t\x12\x0b\n\x03seq\x18\x02 \x01(\x03\x12\x46\n\x08metadata\x18\n \x01(\x0b\x32\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata\x12\x42\n\x06packet\x18\x14 \x01(\x0b\x32\x32.com.webank.ai.eggroll.api.networking.proxy.Packet\x12\x0c\n\x04\x64\x65sc\x18\x1e \x01(\t*O\n\tOperation\x12\t\n\x05START\x10\x00\x12\x07\n\x03RUN\x10\x01\x12\x08\n\x04STOP\x10\x02\x12\x08\n\x04KILL\x10\x03\x12\x0c\n\x08GET_DATA\x10\x04\x12\x0c\n\x08PUT_DATA\x10\x05\x32\xf6\x03\n\x13\x44\x61taTransferService\x12r\n\x04push\x12\x32.com.webank.ai.eggroll.api.networking.proxy.Packet\x1a\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata(\x01\x12r\n\x04pull\x12\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata\x1a\x32.com.webank.ai.eggroll.api.networking.proxy.Packet0\x01\x12s\n\tunaryCall\x12\x32.com.webank.ai.eggroll.api.networking.proxy.Packet\x1a\x32.com.webank.ai.eggroll.api.networking.proxy.Packet\x12\x81\x01\n\x07polling\x12\x38.com.webank.ai.eggroll.api.networking.proxy.PollingFrame\x1a\x38.com.webank.ai.eggroll.api.networking.proxy.PollingFrame(\x01\x30\x01\x32t\n\x0cRouteService\x12\x64\n\x05query\x12\x31.com.webank.ai.eggroll.api.networking.proxy.Topic\x1a(.com.webank.ai.eggroll.api.core.Endpointb\x06proto3' +) + +_OPERATION = DESCRIPTOR.enum_types_by_name["Operation"] +Operation = enum_type_wrapper.EnumTypeWrapper(_OPERATION) +START = 0 +RUN = 1 +STOP = 2 +KILL = 3 +GET_DATA = 4 +PUT_DATA = 5 + + +_MODEL = DESCRIPTOR.message_types_by_name["Model"] +_TASK = DESCRIPTOR.message_types_by_name["Task"] +_TOPIC = DESCRIPTOR.message_types_by_name["Topic"] +_COMMAND = DESCRIPTOR.message_types_by_name["Command"] +_CONF = DESCRIPTOR.message_types_by_name["Conf"] +_METADATA = DESCRIPTOR.message_types_by_name["Metadata"] +_DATA = DESCRIPTOR.message_types_by_name["Data"] +_PACKET = DESCRIPTOR.message_types_by_name["Packet"] +_HEARTBEATRESPONSE = DESCRIPTOR.message_types_by_name["HeartbeatResponse"] +_POLLINGFRAME = DESCRIPTOR.message_types_by_name["PollingFrame"] +Model = _reflection.GeneratedProtocolMessageType( + "Model", + (_message.Message,), + { + "DESCRIPTOR": _MODEL, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Model) + }, +) +_sym_db.RegisterMessage(Model) + +Task = _reflection.GeneratedProtocolMessageType( + "Task", + (_message.Message,), + { + "DESCRIPTOR": _TASK, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Task) + }, +) +_sym_db.RegisterMessage(Task) + +Topic = _reflection.GeneratedProtocolMessageType( + "Topic", + (_message.Message,), + { + "DESCRIPTOR": _TOPIC, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Topic) + }, +) +_sym_db.RegisterMessage(Topic) + +Command = _reflection.GeneratedProtocolMessageType( + "Command", + (_message.Message,), + { + "DESCRIPTOR": _COMMAND, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Command) + }, +) +_sym_db.RegisterMessage(Command) + +Conf = _reflection.GeneratedProtocolMessageType( + "Conf", + (_message.Message,), + { + "DESCRIPTOR": _CONF, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Conf) + }, +) +_sym_db.RegisterMessage(Conf) + +Metadata = _reflection.GeneratedProtocolMessageType( + "Metadata", + (_message.Message,), + { + "DESCRIPTOR": _METADATA, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Metadata) + }, +) +_sym_db.RegisterMessage(Metadata) + +Data = _reflection.GeneratedProtocolMessageType( + "Data", + (_message.Message,), + { + "DESCRIPTOR": _DATA, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Data) + }, +) +_sym_db.RegisterMessage(Data) + +Packet = _reflection.GeneratedProtocolMessageType( + "Packet", + (_message.Message,), + { + "DESCRIPTOR": _PACKET, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Packet) + }, +) +_sym_db.RegisterMessage(Packet) + +HeartbeatResponse = _reflection.GeneratedProtocolMessageType( + "HeartbeatResponse", + (_message.Message,), + { + "DESCRIPTOR": _HEARTBEATRESPONSE, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.HeartbeatResponse) + }, +) +_sym_db.RegisterMessage(HeartbeatResponse) + +PollingFrame = _reflection.GeneratedProtocolMessageType( + "PollingFrame", + (_message.Message,), + { + "DESCRIPTOR": _POLLINGFRAME, + "__module__": "proxy_pb2" + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.PollingFrame) + }, +) +_sym_db.RegisterMessage(PollingFrame) + +_DATATRANSFERSERVICE = DESCRIPTOR.services_by_name["DataTransferService"] +_ROUTESERVICE = DESCRIPTOR.services_by_name["RouteService"] +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _OPERATION._serialized_start = 1420 + _OPERATION._serialized_end = 1499 + _MODEL._serialized_start = 77 + _MODEL._serialized_end = 115 + _TASK._serialized_start = 117 + _TASK._serialized_end = 205 + _TOPIC._serialized_start = 207 + _TOPIC._serialized_end = 319 + _COMMAND._serialized_start = 321 + _COMMAND._serialized_end = 344 + _CONF._serialized_start = 346 + _CONF._serialized_end = 458 + _METADATA._serialized_start = 461 + _METADATA._serialized_end = 871 + _DATA._serialized_start = 873 + _DATA._serialized_end = 907 + _PACKET._serialized_start = 910 + _PACKET._serialized_end = 1052 + _HEARTBEATRESPONSE._serialized_start = 1055 + _HEARTBEATRESPONSE._serialized_end = 1218 + _POLLINGFRAME._serialized_start = 1221 + _POLLINGFRAME._serialized_end = 1418 + _DATATRANSFERSERVICE._serialized_start = 1502 + _DATATRANSFERSERVICE._serialized_end = 2004 + _ROUTESERVICE._serialized_start = 2006 + _ROUTESERVICE._serialized_end = 2122 +# @@protoc_insertion_point(module_scope) diff --git a/python/ofx/api/proto/rollsite/proxy_pb2_grpc.py b/python/ofx/api/proto/rollsite/proxy_pb2_grpc.py new file mode 100644 index 000000000..9a6dbbad9 --- /dev/null +++ b/python/ofx/api/proto/rollsite/proxy_pb2_grpc.py @@ -0,0 +1,303 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import basic_meta_pb2 as basic__meta__pb2 +import grpc +import proxy_pb2 as proxy__pb2 + + +class DataTransferServiceStub(object): + """data transfer service""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.push = channel.stream_unary( + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/push", + request_serializer=proxy__pb2.Packet.SerializeToString, + response_deserializer=proxy__pb2.Metadata.FromString, + ) + self.pull = channel.unary_stream( + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/pull", + request_serializer=proxy__pb2.Metadata.SerializeToString, + response_deserializer=proxy__pb2.Packet.FromString, + ) + self.unaryCall = channel.unary_unary( + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/unaryCall", + request_serializer=proxy__pb2.Packet.SerializeToString, + response_deserializer=proxy__pb2.Packet.FromString, + ) + self.polling = channel.stream_stream( + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/polling", + request_serializer=proxy__pb2.PollingFrame.SerializeToString, + response_deserializer=proxy__pb2.PollingFrame.FromString, + ) + + +class DataTransferServiceServicer(object): + """data transfer service""" + + def push(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def pull(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def unaryCall(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + def polling(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + +def add_DataTransferServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + "push": grpc.stream_unary_rpc_method_handler( + servicer.push, + request_deserializer=proxy__pb2.Packet.FromString, + response_serializer=proxy__pb2.Metadata.SerializeToString, + ), + "pull": grpc.unary_stream_rpc_method_handler( + servicer.pull, + request_deserializer=proxy__pb2.Metadata.FromString, + response_serializer=proxy__pb2.Packet.SerializeToString, + ), + "unaryCall": grpc.unary_unary_rpc_method_handler( + servicer.unaryCall, + request_deserializer=proxy__pb2.Packet.FromString, + response_serializer=proxy__pb2.Packet.SerializeToString, + ), + "polling": grpc.stream_stream_rpc_method_handler( + servicer.polling, + request_deserializer=proxy__pb2.PollingFrame.FromString, + response_serializer=proxy__pb2.PollingFrame.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + "com.webank.ai.eggroll.api.networking.proxy.DataTransferService", + rpc_method_handlers, + ) + server.add_generic_rpc_handlers((generic_handler,)) + + +# This class is part of an EXPERIMENTAL API. +class DataTransferService(object): + """data transfer service""" + + @staticmethod + def push( + request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.stream_unary( + request_iterator, + target, + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/push", + proxy__pb2.Packet.SerializeToString, + proxy__pb2.Metadata.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def pull( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_stream( + request, + target, + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/pull", + proxy__pb2.Metadata.SerializeToString, + proxy__pb2.Packet.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def unaryCall( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/unaryCall", + proxy__pb2.Packet.SerializeToString, + proxy__pb2.Packet.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + @staticmethod + def polling( + request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.stream_stream( + request_iterator, + target, + "/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/polling", + proxy__pb2.PollingFrame.SerializeToString, + proxy__pb2.PollingFrame.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) + + +class RouteServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.query = channel.unary_unary( + "/com.webank.ai.eggroll.api.networking.proxy.RouteService/query", + request_serializer=proxy__pb2.Topic.SerializeToString, + response_deserializer=basic__meta__pb2.Endpoint.FromString, + ) + + +class RouteServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def query(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details("Method not implemented!") + raise NotImplementedError("Method not implemented!") + + +def add_RouteServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + "query": grpc.unary_unary_rpc_method_handler( + servicer.query, + request_deserializer=proxy__pb2.Topic.FromString, + response_serializer=basic__meta__pb2.Endpoint.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + "com.webank.ai.eggroll.api.networking.proxy.RouteService", rpc_method_handlers + ) + server.add_generic_rpc_handlers((generic_handler,)) + + +# This class is part of an EXPERIMENTAL API. +class RouteService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def query( + request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None, + ): + return grpc.experimental.unary_unary( + request, + target, + "/com.webank.ai.eggroll.api.networking.proxy.RouteService/query", + proxy__pb2.Topic.SerializeToString, + basic__meta__pb2.Endpoint.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + ) diff --git a/python/ofx/api/utils/__init__.py b/python/ofx/api/utils/__init__.py new file mode 100644 index 000000000..ae946a49c --- /dev/null +++ b/python/ofx/api/utils/__init__.py @@ -0,0 +1,14 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/ofx/api/utils/grpc_utils.py b/python/ofx/api/utils/grpc_utils.py new file mode 100644 index 000000000..42be68316 --- /dev/null +++ b/python/ofx/api/utils/grpc_utils.py @@ -0,0 +1,53 @@ +# +# Copyright 2019 The FATE Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json + +import grpc + +from ..proto.rollsite import basic_meta_pb2, proxy_pb2, proxy_pb2_grpc + + +def wrap_proxy_grpc_packet(json_body, http_method, url, src_party_id, dst_party_id, job_id=None, headers=None, + overall_timeout=None, role="fateflow", source_host=None, source_port=None): + if not headers: + headers = {} + _src_end_point = basic_meta_pb2.Endpoint(ip=source_host, port=source_port) + _src = proxy_pb2.Topic(name=job_id, partyId="{}".format(src_party_id), role=role, + callback=_src_end_point) + _dst = proxy_pb2.Topic(name=job_id, partyId="{}".format(dst_party_id), role=role, callback=None) + _model = proxy_pb2.Model(name="headers", dataKey=json.dumps(headers)) + _task = proxy_pb2.Task(taskId=job_id, model=_model) + _command = proxy_pb2.Command(name=url) + _conf = proxy_pb2.Conf(overallTimeout=overall_timeout) + _meta = proxy_pb2.Metadata(src=_src, dst=_dst, task=_task, command=_command, operator=http_method, conf=_conf) + _data = proxy_pb2.Data(key=url, value=bytes(json.dumps(json_body), 'utf-8')) + return proxy_pb2.Packet(header=_meta, body=_data) + + +def get_proxy_channel(host, port): + channel = grpc.insecure_channel(f"{host}:{port}") + stub = proxy_pb2_grpc.DataTransferServiceStub(channel) + return channel, stub + + +def gen_routing_metadata(src_party_id, dest_party_id): + routing_head = ( + ("service", "fateflow"), + ("src-party-id", str(src_party_id)), + ("src-role", "guest"), + ("dest-party-id", str(dest_party_id)), + ("dest-role", "host"), + ) + return routing_head \ No newline at end of file diff --git a/python/requirements-container.txt b/python/requirements-container.txt new file mode 100644 index 000000000..d3e9cd4e4 --- /dev/null +++ b/python/requirements-container.txt @@ -0,0 +1,2 @@ +docker +kubernetes \ No newline at end of file diff --git a/python/requirements-docker.txt b/python/requirements-docker.txt new file mode 100644 index 000000000..d58644821 --- /dev/null +++ b/python/requirements-docker.txt @@ -0,0 +1,8 @@ +# fate flow +-r requirements-flow.txt + +# fate +-r requirements-fate.txt + +# container +-r requirements-container.txt diff --git a/python/requirements-eggroll.txt b/python/requirements-eggroll.txt new file mode 100644 index 000000000..38e9ade97 --- /dev/null +++ b/python/requirements-eggroll.txt @@ -0,0 +1,6 @@ +cloudpickle==2.1.0 +lmdb==1.3.0 +protobuf==4.24.4 +grpcio==1.59.3 +grpcio-tools==1.59.3 +psutil>=5.7.0 \ No newline at end of file diff --git a/python/requirements-fate.txt b/python/requirements-fate.txt new file mode 100644 index 000000000..283df32e6 --- /dev/null +++ b/python/requirements-fate.txt @@ -0,0 +1,22 @@ +lmdb==1.3.0 +fate_utils +pydantic==1.10.12 +cloudpickle==2.1.0 +click +ruamel-yaml==0.16 +numpy +pandas==2.0.3 +transformers==4.30.2 +accelerate==0.20.2 +beautifultable +requests<2.26.0 +scikit-learn +omegaconf +rich +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp-proto-grpc +mmh3==3.0.0 +protobuf==4.24.4 +grpcio==1.59.3 +safetensors==0.4.1 \ No newline at end of file diff --git a/python/requirements-flow.txt b/python/requirements-flow.txt new file mode 100644 index 000000000..62c704cb5 --- /dev/null +++ b/python/requirements-flow.txt @@ -0,0 +1,26 @@ +pip>=21 +apsw<=3.10 +Flask==2.2.5 +grpcio==1.59.3 +grpcio-tools==1.59.3 +requests<2.26.0 +urllib3==1.26.18 +ruamel-yaml==0.16 +cachetools==3.0.0 +filelock==3.3.1 +pydantic==1.10.12 +webargs +peewee==3.9.3 +python-dotenv==0.13.0 +pyyaml==5.4.1 +networkx +psutil>=5.7.0 +casbin_peewee_adapter +casbin +pymysql +kazoo +shortuuid +cos-python-sdk-v5==1.9.10 +typing-extensions==4.5.0 +boto3 +pyarrow==14.0.1 diff --git a/python/requirements-pulsar.txt b/python/requirements-pulsar.txt new file mode 100644 index 000000000..03726430b --- /dev/null +++ b/python/requirements-pulsar.txt @@ -0,0 +1 @@ +pulsar-client==2.10.2 \ No newline at end of file diff --git a/python/requirements-rabbitmq.txt b/python/requirements-rabbitmq.txt new file mode 100644 index 000000000..96aeb3111 --- /dev/null +++ b/python/requirements-rabbitmq.txt @@ -0,0 +1 @@ +pika==1.2.1 \ No newline at end of file diff --git a/python/requirements-spark.txt b/python/requirements-spark.txt new file mode 100644 index 000000000..da3b4b1a9 --- /dev/null +++ b/python/requirements-spark.txt @@ -0,0 +1 @@ +pyspark \ No newline at end of file diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 000000000..786714ebd --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1,17 @@ +# fate flow +-r requirements-flow.txt + +# container +-r requirements-container.txt + +# eggroll +-r requirements-eggroll.txt + +# rabbitmq +-r requirements-rabbitmq.txt + +# pulsar +-r requirements-pulsar.txt + +# spark +-r requirements-spark.txt diff --git a/python/setup.py b/python/setup.py new file mode 100644 index 000000000..51cb621a2 --- /dev/null +++ b/python/setup.py @@ -0,0 +1,107 @@ +import os +import shutil + +import fate_flow +from setuptools import find_packages, setup, Command + +packages = find_packages(".") +install_requires = [ + "apsw", + "Flask", + "grpcio", + "grpcio-tools", + "requests", + "urllib3", + "cachetools", + "filelock", + "pydantic", + "webargs", + "peewee", + "python-dotenv", + "pyyaml", + "networkx", + "psutil", + "casbin_peewee_adapter", + "casbin", + "pymysql", + "kazoo", + "shortuuid", + "cos-python-sdk-v5", + "typing-extensions", + "ruamel-yaml==0.16", + "boto3" +] +extras_require = { + "rabbitmq": ["pika==1.2.1"], + "pulsar": ["pulsar-client==2.10.2"], + "spark": ["pyspark"], + "eggroll": [ + "cloudpickle", + "lmdb", + "protobuf", + "grpcio", + "grpcio-tools", + "protobuf", + ], + "all": ["fate_flow[rabbitmq,pulsar,spark,eggroll]"], +} + + +CONF_NAME = "conf" +PACKAGE_NAME = "fate_flow" +ENV_NAME = "fateflow.env" +HOME = os.path.abspath("../") +CONF_PATH = os.path.join(HOME, CONF_NAME) +PACKAGE_CONF_PATH = os.path.join(HOME, "python", "fate_flow", CONF_NAME) +ENV_PATH = os.path.join(HOME, ENV_NAME) +PACKAGE_ENV_PATH = os.path.join(HOME, "python", "fate_flow", ENV_NAME) + +readme_path = os.path.join(HOME, "README.md") + +entry_points = {"console_scripts": ["fate_flow = fate_flow.commands.server_cli:flow_server_cli"]} + +if os.path.exists(readme_path): + with open(readme_path, "r", encoding='utf-8') as f: + long_description = f.read() +else: + long_description = "fate flow" + + +class InstallCommand(Command): + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + if os.path.exists(PACKAGE_CONF_PATH): + shutil.rmtree(PACKAGE_CONF_PATH) + shutil.copytree(CONF_PATH, PACKAGE_CONF_PATH) + shutil.copyfile(ENV_PATH, PACKAGE_ENV_PATH) + + +setup( + name="fate_flow", + version=fate_flow.__version__, + keywords=["federated learning scheduler"], + author="FederatedAI", + author_email="contact@FedAI.org", + long_description_content_type="text/markdown", + long_description=long_description, + license="Apache-2.0 License", + url="https://fate.fedai.org/", + packages=packages, + install_requires=install_requires, + extras_require=extras_require, + package_data={ + "fate_flow": [f"{CONF_NAME}/*", ENV_NAME, "commands/*"] + }, + python_requires=">=3.8", + cmdclass={ + "pre_install": InstallCommand, + }, + entry_points=entry_points +)