Merge pull request #720 from FederatedAI/dev-3.1.0

Dev 3.1.0

BUILD_INFO

@@ -1 +1 @@
-eggroll.version=3.0.0
+eggroll.version=3.1.0

bin/gpu/nvidia.sh

@@ -1,3 +1,3 @@
-result=$(nvidia-smi --query-gpu=name --format=csv, noheader|grep 'NVIDIA'|wc -l)
+result=$(python $EGGROLL_HOME/python/eggroll/deepspeed/utils/gpu_count.py)
 echo  $result
 

conf/whitelist.json

@@ -0,0 +1,138 @@
+{
+    "builtins": [
+        "int",
+        "list",
+        "set"
+    ],
+    "collections": [
+        "defaultdict",
+        "OrderedDict"
+    ],
+    "eggroll.core.transfer_model": [
+        "ErRollSiteHeader"
+    ],
+    "eggroll.roll_pair.task.storage": [
+        "BSS"
+    ],
+    "federatedml.cipher_compressor.compressor": [
+        "PackingCipherTensor",
+        "NormalCipherPackage",
+        "PackingCipherTensorPackage"
+    ],
+    "federatedml.ensemble.basic_algorithms.decision_tree.tree_core.feature_histogram": [
+        "HistogramBag",
+        "FeatureHistogramWeights"
+    ],
+    "federatedml.ensemble.basic_algorithms.decision_tree.tree_core.feature_importance": [
+        "FeatureImportance"
+    ],
+    "federatedml.ensemble.basic_algorithms.decision_tree.tree_core.g_h_optim": [
+        "SplitInfoPackage",
+        "SplitInfoPackage2"
+    ],
+    "federatedml.ensemble.basic_algorithms.decision_tree.tree_core.node": [
+        "Node"
+    ],
+    "federatedml.ensemble.basic_algorithms.decision_tree.tree_core.splitter": [
+        "SplitInfo"
+    ],
+    "federatedml.evaluation.performance_recorder": [
+        "PerformanceRecorder"
+    ],
+    "federatedml.feature.binning.bin_result": [
+        "BinColResults"
+    ],
+    "federatedml.feature.binning.optimal_binning.bucket_info": [
+        "Bucket"
+    ],
+    "federatedml.feature.binning.optimal_binning.heap": [
+        "MinHeap",
+        "IvHeapNode",
+        "GiniHeapNode",
+        "ChiSquareHeapNode"
+    ],
+    "federatedml.feature.binning.quantile_summaries": [
+        "SparseQuantileSummaries",
+        "Stats",
+        "QuantileSummaries"
+    ],
+    "federatedml.feature.fate_element_type": [
+        "NoneType"
+    ],
+    "federatedml.feature.homo_feature_binning.homo_binning_base": [
+        "SplitPointNode"
+    ],
+    "federatedml.feature.instance": [
+        "Instance"
+    ],
+    "federatedml.feature.one_hot_encoder": [
+        "TransferPair"
+    ],
+    "federatedml.feature.sparse_vector": [
+        "SparseVector"
+    ],
+    "federatedml.framework.weights": [
+        "NumpyWeights",
+        "TransferableWeights",
+        "NumericWeights",
+        "ListWeights",
+        "DictWeights",
+        "OrderDictWeights"
+    ],
+    "federatedml.linear_model.linear_model_weight": [
+        "LinearModelWeights"
+    ],
+    "federatedml.secureprotol.fate_paillier": [
+        "PaillierPublicKey",
+        "PaillierEncryptedNumber"
+    ],
+    "federatedml.secureprotol.fixedpoint": [
+        "FixedPointNumber"
+    ],
+    "federatedml.secureprotol.number_theory.field.integers_modulo_prime_field": [
+        "IntegersModuloPrimeElement"
+    ],
+    "federatedml.secureprotol.number_theory.group.twisted_edwards_curve_group": [
+        "TwistedEdwardsCurveElement"
+    ],
+    "federatedml.secureprotol.symmetric_encryption.cryptor_executor": [
+        "CryptoExecutor"
+    ],
+    "federatedml.secureprotol.symmetric_encryption.pohlig_hellman_encryption": [
+        "PohligHellmanCiphertext",
+        "PohligHellmanCipherKey"
+    ],
+    "federatedml.statistic.intersect.intersect_preprocess": [
+        "BitArray"
+    ],
+    "federatedml.statistic.statics": [
+        "SummaryStatistics"
+    ],
+    "gmpy2": [
+        "from_binary"
+    ],
+    "numpy": [
+        "ndarray",
+        "dtype"
+    ],
+    "numpy.core.multiarray": [
+        "scalar",
+        "_reconstruct"
+    ],
+    "numpy.core.numeric": [
+        "_frombuffer"
+    ],
+    "tensorflow.python.framework.ops": [
+        "convert_to_tensor"
+    ],
+    "torch._utils": [
+        "_rebuild_tensor_v2"
+    ],
+    "ipcl_python.bindings.ipcl_bindings": [
+        "ipclPublicKey"
+    ],
+    "ipcl_python.ipcl_python": [
+        "PaillierPublicKey",
+        "PaillierEncryptedNumber"
+    ]
+}

doc/Eggroll部署文档说明.md

@@ -21,7 +21,7 @@
 从github拉取Eggroll项目，通过执行auto-packaging.sh自动打包脚本在同目录下生成eggroll.tar.gz
 
 ```shell
-git clone -b v3.0.0 https://github.com/FederatedAI/eggroll.git
+git clone -b v3.1.0 https://github.com/FederatedAI/eggroll.git
 cd eggroll/deploy
 windows : auto-packaging.bat
 linux : sh auto-packaging.sh

doc/upgrade_helper_guide.md

@@ -89,7 +89,7 @@ ${MYSQL_HOME_PATH}/bin/mysqldump -h <mysql-host> -u <username> -p<passwd> -P <po
 
 > 获取升级脚本
 
-[升级脚本](https://github.com/WeBankFinTech/eggroll/blob/dev-3.0.0-rc/deploy/upgrade_helper.py)
+[升级脚本](https://github.com/WeBankFinTech/eggroll/blob/dev-3.1.0/deploy/upgrade_helper.py)
 
 ```
 export PKG_BASE_PATH={your upgrade eggroll version eggroll home dir}

jvm/cluster_dashboard/pom.xml

@@ -6,7 +6,7 @@
         <artifactId>eggroll-all</artifactId>
         <groupId>org.fedai.eggroll</groupId>
         <relativePath>../pom.xml</relativePath>
-        <version>3.0.0</version>
+        <version>3.1.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>cluster_dashboard</artifactId>

jvm/cluster_manager/pom.xml

@@ -6,7 +6,7 @@
         <groupId>org.fedai.eggroll</groupId>
         <artifactId>eggroll-all</artifactId>
         <relativePath>../pom.xml</relativePath>
-        <version>3.0.0</version>
+        <version>3.1.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>cluster_manager</artifactId>

jvm/cluster_manager/src/test/resources/application.properties

@@ -16,7 +16,7 @@
 
 [eggroll]
 application.title=eggroll
-application.version=3.0.0
+application.version=3.1.0
 # core
 #eggroll.resourcemanager.clustermanager.jdbc.driver.class.name=org.h2.Driver
 eggroll.resourcemanager.clustermanager.jdbc.driver.class.name=com.mysql.cj.jdbc.Driver

jvm/core/pom.xml

@@ -6,7 +6,7 @@
         <groupId>org.fedai.eggroll</groupId>
         <artifactId>eggroll-all</artifactId>
         <relativePath>../pom.xml</relativePath>
-        <version>3.0.0</version>
+        <version>3.1.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>core</artifactId>

jvm/node_manager/pom.xml

@@ -6,7 +6,7 @@
         <artifactId>eggroll-all</artifactId>
         <groupId>org.fedai.eggroll</groupId>
         <relativePath>../pom.xml</relativePath>
-        <version>3.0.0</version>
+        <version>3.1.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>node_manager</artifactId>

jvm/pom.xml

@@ -23,7 +23,7 @@
 
     <groupId>org.fedai.eggroll</groupId>
     <artifactId>eggroll-all</artifactId>
-    <version>3.0.0</version>
+    <version>3.1.0</version>
     <name>Eggroll Project</name>
 
     <licenses>
@@ -145,7 +145,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <eggroll.version>3.0.0</eggroll.version>
+        <eggroll.version>3.1.0</eggroll.version>
 
         <!-- Languages -->
         <code.cache.size>512m</code.cache.size>

python/eggroll/__init__.py

@@ -14,4 +14,11 @@
 #  limitations under the License.
 #
 
-__version__ = "3.0.0"
+__version__ = "3.1.0"
+
+import os
+
+if os.environ.get("EGGROLL_2X_BACKPORT"):
+    from .backport import backport_patch
+
+    backport_patch()

python/eggroll/backport/README.md

@@ -0,0 +1,5 @@
+## Backport of Eggroll-2.x
+
+This directory contains the backport api of Eggroll-2.x and may be used for FATE-1.x.
+For users who use newer version api of Eggroll-3.x, 
+which is not compatible with FATE-2.x, this directory can be safely deleted.

python/eggroll/backport/__init__.py

@@ -0,0 +1,22 @@
+#  Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#
+
+
+def backport_patch():
+    import sys
+    from ._package_loader_patch import Eggroll2xPackageFinder
+
+    sys.meta_path.insert(0, Eggroll2xPackageFinder())

python/eggroll/backport/_module_rollpair_patch.py

@@ -0,0 +1,23 @@
+#  Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#
+
+from ._wrap_rollpair import (
+    WrappedRp as RollPair,
+    WrappedRpc as RollPairContext,
+    runtime_init,
+)
+
+__all__ = ["RollPair", "RollPairContext", "runtime_init"]

python/eggroll/backport/_module_rollsite_patch.py

@@ -0,0 +1,19 @@
+#  Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#
+
+from ._wrap_rollsite import WrappedRollSiteContext as RollSiteContext
+
+__all__ = ["RollSiteContext"]

python/eggroll/backport/_module_session_patch.py

@@ -0,0 +1,19 @@
+#  Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#
+
+from ._wrap_session import session_init
+
+__all__ = ["session_init"]