Merge pull request #495 from FederatedAI/dev-2.0-beta

v2.0.0-beta

.github/workflows/doc.yml

@@ -4,7 +4,8 @@ on:
  push:
  branches:
  - 'main'
- - 'develop-[0-9].[0-9].[0-9]'
+ - 'develop-[0-9]+.[0-9]+.[0-9]+'
+ - 'build-doc-[0-9]+.[0-9]+.[0-9]+-[a-zA-Z]+'
 
  schedule:
  - cron: '0 8 * * *'
@@ -41,6 +42,7 @@ jobs:
  VERSION='${{ github.ref_name }}'
  [ "$VERSION" == main ] && { VERSION=latest; ALIAS='main master'; }
  VERSION="${VERSION#develop-}"
+ VERSION="${VERSION#build-doc-}"
 
  mike deploy --push --update-aliases "$VERSION" $ALIAS
  mike set-default --push latest

.gitignore

@@ -17,12 +17,11 @@ venv
 /logs/
 /jobs/
 /audit/
+/localfs/
 .vscode/*
 /temp/
 /tmp
 /worker/
-/provider_registrar/
-/model_local_cache/
 *.db
 *.db-journal
 *.whl
@@ -32,3 +31,14 @@ venv
 
 # doc
 /site/
+
+/python/fate_flow/data
+/python/fate_flow/model
+/python/fate_flow/logs
+/python/fate_flow/jobs
+/python/fate_flow/localfs
+/python/fate_flow/*.env
+/python/fate_flow/conf
+/python/build
+/python/dist
+/python/*.egg-info

README.md

@@ -20,8 +20,6 @@ Providing production-level service capabilities:
 - High Availability
 - CLI, REST API, Python API
 
-For detailed introduction, please refer to [FATE Flow Overall Design](https://federatedai.github.io/FATE-Flow/latest/fate_flow/#overall-design)
-
 ## Deployment
 
 Please refer to [FATE](https://github.com/FederatedAI/FATE)

README.zh.md

@@ -4,7 +4,7 @@
 
 FATE Flow是一个联邦学习端到端全流程的多方联合任务安全调度平台, 基于:
 
-- [共享状态调度架构](https://storage.googleapis.com/pub-tools-public-publication-data/pdf/41684.pdf)
+- 共享状态调度架构
 - 跨数据中心的多方安全通信
 
 提供生产级服务能力:
@@ -20,7 +20,6 @@ FATE Flow是一个联邦学习端到端全流程的多方联合任务安全调
 - 系统高可用
 - CLI、REST API、Python API
 
-详细介绍请参考[FATE Flow整体设计](https://federatedai.github.io/FATE-Flow/latest/zh/fate_flow/)
 
 ## 部署
 

RELEASE.md

@@ -1,3 +1,14 @@
+## Release 2.0.0-beta
+### Major Features and Improvements
+* Migrated functions: data upload/download, process scheduling, component output data/model/metric management, multi-storage adaptation for models, authentication, authorization, feature anonymization, multi-computing/storage/communication engine adaptation, and system high availability
+* Optimized process scheduling, with scheduling separated and customizable, and added priority scheduling
+* Optimized algorithm component scheduling, dividing execution steps into preprocessing, running, and post-processing
+* Optimized multi-version algorithm component registration, supporting registration for mode of components
+* Optimized client authentication logic, supporting permission management for multiple clients
+* Optimized RESTful interface, making parameter fields and types, return fields, and status codes clearer
+* Decoupling the system layer from the algorithm layer, with system configuration moved from the FATE repository to the Flow repository
+* Published FATE Flow package to PyPI and added service-level CLI for service management
+
 ## Release 2.0.0-alpha
 ### Feature Highlights
 * Adapted to new scalable and standardized federated DSL IR

bin/init_env.sh

@@ -18,10 +18,11 @@
 
 fate_project_base=$(cd `dirname "$(realpath "${BASH_SOURCE[0]:-${(%):-%x}}")"`; cd ../;cd ../;pwd)
 export FATE_PROJECT_BASE=$fate_project_base
-export FATE_DEPLOY_BASE=$fate_project_base
+export FATE_PYTHONPATH=
 export EGGROLL_HOME=
 export PYTHONPATH=
 export FATE_ENV=
+export SPARK_HOME=
 
 export FATE_LOG_LEVEL=DEBUG
 

bin/service.sh

@@ -28,7 +28,7 @@ echo "PROJECT_BASE: "${PROJECT_BASE}
 INI_ENV_SCRIPT=${FATE_FLOW_BASE}/bin/init_env.sh
 echo $INI_ENV_SCRIPT
 if test -f "${INI_ENV_SCRIPT}"; then
- source ${$INI_ENV_SCRIPT}/bin/init_env.sh
+ source $INI_ENV_SCRIPT
  echo "PYTHONPATH: "${PYTHONPATH}
 else
  echo "file not found: ${INI_ENV_SCRIPT}"

conf/casbin_model.conf

@@ -0,0 +1,14 @@
+[request_definition]
+r = sub, obj, act
+
+[policy_definition]
+p = sub, obj, act
+
+[role_definition]
+g = _, _
+
+[policy_effect]
+e = some(where (p.eft == allow))
+
+[matchers]
+m = g(r.sub, p.sub) && r.obj == p.obj && r.act == p.act

conf/job_default_config.yaml

@@ -1,45 +1,27 @@
 # resource
-total_cores_overweight_percent: 1 # 1 means no overweight
-total_memory_overweight_percent: 1 # 1 means no overweight
-task_parallelism: 1
-task_cores: 4
-task_memory: 0 # mb
-max_cores_percent_per_job: 1 # 1 means total
-
-# scheduling
+job_cores: 4
+computing_partitions: 8
+task_run:
+ spark:
+ num-executors: 2
+ executor-cores: 2
+ eggroll:
+ eggroll.session.processors.per.node: 4
+ standalone:
+ cores: 4
 job_timeout: 259200 # s
 remote_request_timeout: 30000 # ms
 federated_command_trys: 3
-end_status_job_scheduling_time_limit: 300000 # ms
-end_status_job_scheduling_updates: 1
-auto_retries: 1
-auto_retry_delay: 1 #seconds
-# It can also be specified in the job configuration using the federated_status_collect_type parameter
-federated_status_collect_type: PUSH
-detect_connect_max_retry_count: 3
-detect_connect_long_retry_count: 2
-
-# upload
-upload_block_max_bytes: 104857600 # bytes
-
-#component output
-output_data_summary_count_limit: 100
-
-task_default_conf:
- logger:
- type: flow
- metadata:
- level: DEBUG
- debug_mode: true
- device:
- type: CPU
- output:
- data:
- type: directory
- format: dataframe
- model:
- type: directory
- format: json
- metric:
- type: directory
- format: json
+auto_retries: 0
+sync_type: callback # poll or callback
+task_logger:
+ type: flow
+ metadata:
+ level: DEBUG
+ debug_mode: true
+task_device:
+ type: CPU
+launcher:
+ deepspeed:
+ timeout: 21600 # s
+ world_size: 2

conf/permission_casbin_model.conf

@@ -0,0 +1,11 @@
+[request_definition]
+r = party_id, type, value
+
+[policy_definition]
+p = party_id, type, value
+
+[policy_effect]
+e = some(where (p.eft == allow))
+
+[matchers]
+m = r.party_id == p.party_id && r.type == p.type && r.value == p.value

conf/pulsar_route_table.yaml

@@ -1,14 +1,14 @@
 9999:
  # host can be a domain like 9999.fate.org
- host: 172.16.153.37
+ host: 127.0.0.1
  port: 6650
  sslPort: 6651
  # set proxy address for this pulsar cluster
  proxy: ""
 
 10000:
  # host can be a domain like 10000.fate.org
- host: 172.16.153.37
+ host: 127.0.0.1
  port: 6650
  sslPort: 6651
  proxy: ""

conf/service_conf.yaml

@@ -1,24 +1,44 @@
-force_use_sqlite: true
-party_id: "10000"
+party_id: "9999"
+use_registry: false
+encrypt:
+ key_0:
+ module: fate_flow.hub.encrypt.password_encrypt#pwdecrypt
+ # base on: fate_flow/conf/
+ private_path: private_key.pem
 fateflow:
- # you must set real ip address, 127.0.0.1 and 0.0.0.0 is not supported
  host: 127.0.0.1
  http_port: 9380
  grpc_port: 9360
  proxy_name: rollsite
+ nginx:
+ host:
+ http_port:
+ grpc_port:
 database:
- name: fate_flow
- user: fate
- passwd: fate
- host: 127.0.0.1
- port: 3306
- max_connections: 100
- stale_timeout: 30
-# engine services
+ engine: sqlite
+ # encrypt passwd key
+ decrypt_key:
+ mysql:
+ name: fate_flow
+ user: fate
+ passwd: fate
+ host: 127.0.0.1
+ port: 3306
+ max_connections: 100
+ stale_timeout: 30
+ sqlite:
+ # default fate_flow/runtime/system_settings: SQLITE_PATH
+ # /xxx/xxx.sqlite
+ path:
 default_engines:
  computing: standalone
  federation: standalone
  storage: standalone
+default_provider:
+ name: fate
+ # version default: fateflow.env
+ version:
+ device: local
 federation:
  pulsar:
  host: 192.168.0.5
@@ -57,27 +77,48 @@ federation:
  port: 9370
 computing:
  standalone:
- cores_per_node: 20
- nodes: 1
+ cores: 32
  eggroll:
- cores_per_node: 16
- nodes: 1
+ cores: 32
+ nodes: 2
  spark:
  # default use SPARK_HOME environment variable
  home:
- cores_per_node: 20
- nodes: 2
-worker:
- type: native
- docker:
- config:
- # https://docker-py.readthedocs.io/en/stable/client.html#docker.client.DockerClient
- base_url: unix:///var/run/docker.sock
- image: ccr.ccs.tencentyun.com/federatedai/fate_algorithm:2.0.0-alpha
- # on container
- fate_root_dir: /data/projects/fate
- # on host
- eggroll_conf_dir:
- k8s:
- image: ccr.ccs.tencentyun.com/federatedai/fate_algorithm:2.0.0-alpha
- namespace: fate-10000
+ cores: 32
+storage:
+ hdfs:
+ name_node: hdfs://fate-cluster
+hook_module:
+ client_authentication: fate_flow.hook.flow.client_authentication
+ site_authentication: fate_flow.hook.flow.site_authentication
+ permission: fate_flow.hook.flow.permission
+authentication:
+ client: false
+ site: false
+ permission: false
+model_store:
+ engine: file
+ # encrypt passwd key
+ decrypt_key:
+ file:
+ # default fate_flow/runtime/system_settings: MODEL_STORE_PATH
+ path:
+ mysql:
+ name: fate_flow
+ user: fate
+ passwd: fate
+ host: 127.0.0.1
+ port: 3306
+ max_connections: 100
+ stale_timeout: 30
+ tencent_cos:
+ Region:
+ SecretId:
+ SecretKey:
+ Bucket:
+zookeeper:
+ hosts:
+ - 127.0.0.1:2181
+ use_acl: true
+ user: fate
+ password: fate