From 1659b3cd84ba53145b87d96b3994b3d5de4d2ca9 Mon Sep 17 00:00:00 2001 From: sagewe Date: Wed, 10 Jan 2024 17:51:38 +0800 Subject: [PATCH 01/20] feat: backport init (#708) create WrappedXX class to mimic old eggroll api Signed-off-by: sagewe --- python/eggroll_2x/__init__.py | 3 + python/eggroll_2x/_partitioner.py | 5 ++ python/eggroll_2x/_serdes.py | 12 +++ python/eggroll_2x/_wrap.py | 117 ++++++++++++++++++++++++++++++ 4 files changed, 137 insertions(+) create mode 100644 python/eggroll_2x/__init__.py create mode 100644 python/eggroll_2x/_partitioner.py create mode 100644 python/eggroll_2x/_serdes.py create mode 100644 python/eggroll_2x/_wrap.py diff --git a/python/eggroll_2x/__init__.py b/python/eggroll_2x/__init__.py new file mode 100644 index 000000000..60da1f253 --- /dev/null +++ b/python/eggroll_2x/__init__.py @@ -0,0 +1,3 @@ +from ._wrap import session_init, runtime_init + +__all__ = ["session_init", "runtime_init"] diff --git a/python/eggroll_2x/_partitioner.py b/python/eggroll_2x/_partitioner.py new file mode 100644 index 000000000..17aaf1f70 --- /dev/null +++ b/python/eggroll_2x/_partitioner.py @@ -0,0 +1,5 @@ +import mmh3 + + +def mmh3_partitioner(key: bytes, total_partitions): + return mmh3.hash(key) % total_partitions diff --git a/python/eggroll_2x/_serdes.py b/python/eggroll_2x/_serdes.py new file mode 100644 index 000000000..00846174a --- /dev/null +++ b/python/eggroll_2x/_serdes.py @@ -0,0 +1,12 @@ +from pickle import dumps as p_dumps +from pickle import loads as p_loads + + +class UnrestrictedSerdes: + @staticmethod + def serialize(obj) -> bytes: + return p_dumps(obj) + + @staticmethod + def deserialize(bytes) -> object: + return p_loads(bytes) diff --git a/python/eggroll_2x/_wrap.py b/python/eggroll_2x/_wrap.py new file mode 100644 index 000000000..9f6cc3239 --- /dev/null +++ b/python/eggroll_2x/_wrap.py @@ -0,0 +1,117 @@ +import uuid + +from eggroll.computing import runtime_init as _runtime_init, RollPairContext, RollPair +from eggroll.computing.tasks.store import StoreTypes +from eggroll.session import session_init as _session_init +from ._partitioner import mmh3_partitioner +from ._serdes import UnrestrictedSerdes + + +def session_init(session_id, options) -> "WrappedSession": + _session = _session_init(session_id=session_id, options=options) + return WrappedSession(session=_session) + + +def runtime_init(session: "WrappedSession") -> "WrappedRpc": + rpc = _runtime_init(session=session._session) + return WrappedRpc(rpc=rpc) + + +class WrappedSession: + def __init__(self, session): + self._session = session + + def get_session_id(self): + return self._session.get_session_id() + + +class WrappedRpc: + def __init__(self, rpc: "RollPairContext"): + self._rpc = rpc + + @property + def session_id(self): + return self._rpc.session.get_session_id() + + def load(self, namespace, name, options): + store_type = options.get("store_type", StoreTypes.ROLLPAIR_LMDB) + return self._rpc.load_rp(namespace=namespace, name=name, store_type=store_type) + + def parallelize(self, data, options: dict = None): + if options is None: + options = {} + namespace = options.get("namespace", None) + name = options.get("name", None) + if namespace is None: + namespace = self.session_id + if name is None: + name = str(uuid.uuid1()) + include_key = options.get("include_key", False) + total_partitions = options.get("total_partitions", 1) + partitioner = mmh3_partitioner + partitioner_type = 0 + key_serdes_type = 0 + value_serdes_type = 0 + store_type = options.get("store_type", StoreTypes.ROLLPAIR_IN_MEMORY) + + # generate data + if include_key: + data = ( + (UnrestrictedSerdes.serialize(key), UnrestrictedSerdes.serialize(value)) + for key, value in data + ) + else: + data = ( + (UnrestrictedSerdes.serialize(i), UnrestrictedSerdes.serialize(value)) + for i, value in enumerate(data) + ) + return WrappedRp( + self._rpc.parallelize( + data=data, + total_partitions=total_partitions, + partitioner=partitioner, + partitioner_type=partitioner_type, + key_serdes_type=key_serdes_type, + value_serdes_type=value_serdes_type, + store_type=store_type, + namespace=namespace, + name=name, + ) + ) + + def cleanup(self, name, namespace): + return self._rpc.cleanup(name=name, namespace=namespace) + + def stop(self): + return self._rpc.session.stop() + + def kill(self): + return self._rpc.session.kill() + + +class WrappedRp: + def __init__(self, rp: RollPair): + self._rp = rp + + def get_partitions(self): + return self._rp.get_partitions() + + def get(self, k, options: dict = None): + if options is None: + options = {} + value = self._rp.get( + k=UnrestrictedSerdes.serialize(k), partitioner=mmh3_partitioner + ) + if value is None: + return None + else: + return UnrestrictedSerdes.deserialize(value) + + def put(self, k, v, options: dict = None): + if options is None: + options = {} + self._rp.put( + k=UnrestrictedSerdes.serialize(k), + v=UnrestrictedSerdes.serialize(v), + partitioner=mmh3_partitioner, + ) From a2f9b1393a8e453c2913e90f2c61260a64b12c17 Mon Sep 17 00:00:00 2001 From: sagewe Date: Wed, 10 Jan 2024 19:49:41 +0800 Subject: [PATCH 02/20] feat: impl some api (#708) - Introduced `random` and `typing` (Callable, Iterable, Any, Tuple) imports for advanced operations and type hinting. - Added comprehensive methods to WrappedRp class including `delete`, `count`, `get_all`, `put_all`, `take`, `first`, `destroy`, and various map-reduce functions. - Implemented serialization and deserialization support using UnrestrictedSerdes for robust data handling. - Extended map-reduce capabilities with custom partition operations, sampling, filtering, repartitioning, and binary operations on sorted partitions. - Included specialized merge functions for union, join, and subtract operations on keys. - Organized and optimized functions with private utility methods for serialization and iteration processing. - Ensured compatibility and efficiency with thoughtful error handling and condition checks. Signed-off-by: sagewe --- python/eggroll_2x/_wrap.py | 531 +++++++++++++++++++++++++++++++++++++ 1 file changed, 531 insertions(+) diff --git a/python/eggroll_2x/_wrap.py b/python/eggroll_2x/_wrap.py index 9f6cc3239..8b2d415b9 100644 --- a/python/eggroll_2x/_wrap.py +++ b/python/eggroll_2x/_wrap.py @@ -1,4 +1,6 @@ +import random import uuid +from typing import Callable, Iterable, Any, Tuple from eggroll.computing import runtime_init as _runtime_init, RollPairContext, RollPair from eggroll.computing.tasks.store import StoreTypes @@ -115,3 +117,532 @@ def put(self, k, v, options: dict = None): v=UnrestrictedSerdes.serialize(v), partitioner=mmh3_partitioner, ) + + def delete(self, k, options: dict = None): + if options is None: + options = {} + self._rp.delete(k=UnrestrictedSerdes.serialize(k), partitioner=mmh3_partitioner) + + def count(self): + return self._rp.count() + + def get_all(self, limit=None, options: dict = None): + for k, v in self._rp.get_all(limit=limit): + yield UnrestrictedSerdes.deserialize(k), UnrestrictedSerdes.deserialize(v) + + def put_all(self, items, output=None, options: dict = None): + if options is None: + options = {} + include_key = options.get("include_key", True) + if include_key: + self._rp.put_all( + ( + (UnrestrictedSerdes.serialize(k), UnrestrictedSerdes.serialize(v)) + for k, v in items + ), + ) + else: + self._rp.put_all( + ( + (UnrestrictedSerdes.serialize(i), UnrestrictedSerdes.serialize(v)) + for i, v in enumerate(items) + ), + ) + + def take(self, n: int, options: dict = None): + keys_only = options.get("keys_only", False) + if keys_only: + return [UnrestrictedSerdes.deserialize(k) for k in self._rp.take(n=n, options=options)] + else: + return [ + (UnrestrictedSerdes.deserialize(k), UnrestrictedSerdes.deserialize(v)) + for k, v in self._rp.take(n=n, options=options) + ] + + def first(self, options: dict = None): + resp = self.take(1, options=options) + if resp: + return resp[0] + else: + return None + + def destroy(self, options: dict = None): + self._rp.destroy() + + def _map_reduce_partitions_with_index( + self, + map_partition_op: Callable[[int, Iterable], Iterable], + reduce_partition_op: Callable[[Any, Any], Any] = None, + shuffle=True, + output_num_partitions=None, + ): + if not shuffle and reduce_partition_op is not None: + raise ValueError("when shuffle is False, it is not allowed to specify reduce_partition_op") + if output_num_partitions is None: + output_num_partitions = self._rp.get_partitions() + return WrappedRp(self._rp.map_reduce_partitions_with_index( + map_partition_op=_lifted_mpwi_map_to_serdes( + map_partition_op, UnrestrictedSerdes, UnrestrictedSerdes, UnrestrictedSerdes, UnrestrictedSerdes, + ), + reduce_partition_op=_lifted_mpwi_reduce_to_serdes(reduce_partition_op, UnrestrictedSerdes), + shuffle=shuffle, + input_key_serdes=UnrestrictedSerdes, + input_key_serdes_type=0, + input_value_serdes=UnrestrictedSerdes, + input_value_serdes_type=0, + input_partitioner=mmh3_partitioner, + input_partitioner_type=0, + output_key_serdes=UnrestrictedSerdes, + output_key_serdes_type=0, + output_value_serdes=UnrestrictedSerdes, + output_value_serdes_type=0, + output_partitioner=mmh3_partitioner, + output_partitioner_type=0, + output_num_partitions=output_num_partitions, + )) + + def map_values(self, func, output=None, options: dict = None): + if options is None: + options = {} + return self._map_reduce_partitions_with_index( + _lifted_map_values_to_mpwi(func), + shuffle=False, + ) + + def map(self, func, output=None, options: dict = None): + if options is None: + options = {} + return self._map_reduce_partitions_with_index( + _lifted_map_to_mpwi(func), + shuffle=False, + ) + + def map_partitions(self, func, reduce_op=None, output=None, options: dict = None): + if options is None: + options = {} + shuffle = options.get('shuffle', True) + return self._map_reduce_partitions_with_index( + _lifted_map_partitions_to_mpwi(func), + reduce_partition_op=reduce_op, + shuffle=shuffle, + ) + + def collapse_partitions(self, func, output=None, options: dict = None): + if options is None: + options = {} + return self._map_reduce_partitions_with_index( + _lifted_apply_partitions_to_mpwi(func), + shuffle=False, + ) + + def map_partitions_with_index(self, func, output=None, options: dict = None): + if options is None: + options = {} + shuffle = options.get('shuffle', True) + return self._map_reduce_partitions_with_index( + map_partition_op=func, + shuffle=shuffle, + ) + + def flat_map(self, func, output=None, options: dict = None): + if options is None: + options = {} + return self._map_reduce_partitions_with_index( + _lifted_flat_map_to_mpwi(func), + shuffle=False, + ) + + def reduce(self, func, output=None, options: dict = None): + if options is None: + options = {} + + return self._rp.reduce( + func=_lifted_reduce_to_serdes(func, UnrestrictedSerdes), + ) + + def sample(self, fraction, seed=None, output=None, options: dict = None): + if options is None: + options = {} + return self._map_reduce_partitions_with_index( + _lifted_sample_to_mpwi(fraction, seed), + shuffle=False, + ) + + def filter(self, func, output=None, options: dict = None): + if options is None: + options = {} + return self._map_reduce_partitions_with_index( + lambda i, x: ((k, v) for k, v in x if func(v)), + shuffle=False, + ) + + @property + def num_partitions(self): + return self._rp.get_partitions() + + def repartition(self, num_partitions) -> "WrappedRp": + if ( + self.num_partitions == num_partitions + ): + return self + + return WrappedRp(self._rp.map_reduce_partitions_with_index( + map_partition_op=lambda i, x: x, + reduce_partition_op=None, + shuffle=True, + input_key_serdes=UnrestrictedSerdes, + input_key_serdes_type=0, + input_value_serdes=UnrestrictedSerdes, + input_value_serdes_type=0, + input_partitioner=mmh3_partitioner, + input_partitioner_type=0, + output_key_serdes=UnrestrictedSerdes, + output_key_serdes_type=0, + output_value_serdes=UnrestrictedSerdes, + output_value_serdes_type=0, + output_partitioner=mmh3_partitioner, + output_partitioner_type=0, + output_num_partitions=num_partitions, + )) + + def repartition_with(self, other: "WrappedRp") -> Tuple["WrappedRp", "WrappedRp"]: + if self._rp.num_partitions == other._rp.num_partitions: + return self, other + if self._rp.num_partitions > other._rp.num_partitions: + return self, other.repartition(self.num_partitions) + else: + return self.repartition(other.num_partitions), other + + def binarySortedMapPartitionsWithIndex( + self, + other: "WrappedRp", + binary_sorted_map_partitions_with_index_op: Callable[[int, Iterable, Iterable], Iterable], + ): + first, second = self.repartition_with(other) + + # apply binary_sorted_map_partitions_with_index_op + return WrappedRp(first._rp.binary_sorted_map_partitions_with_index( + other=second._rp, + binary_map_partitions_with_index_op=_lifted_sorted_binary_map_partitions_with_index_to_serdes( + binary_sorted_map_partitions_with_index_op, + UnrestrictedSerdes, + UnrestrictedSerdes, + UnrestrictedSerdes, + ), + key_serdes=UnrestrictedSerdes, + key_serdes_type=0, + partitioner=mmh3_partitioner, + partitioner_type=0, + first_input_value_serdes=UnrestrictedSerdes, + first_input_value_serdes_type=0, + second_input_value_serdes=UnrestrictedSerdes, + second_input_value_serdes_type=0, + output_value_serdes=UnrestrictedSerdes, + output_value_serdes_type=0, + )) + + def subtract_by_key(self, other, output=None, options: dict = None): + if options is None: + options = {} + return self.binarySortedMapPartitionsWithIndex( + other=other, + binary_sorted_map_partitions_with_index_op=_lifted_subtract_by_key_to_sbmpwi(), + ) + + def union(self, other, func=lambda v1, v2: v1, output=None, options: dict = None): + if options is None: + options = {} + return self.binarySortedMapPartitionsWithIndex( + other=other, + binary_sorted_map_partitions_with_index_op=_lifted_union_merge_to_sbmpwi(func), + ) + + def join(self, other, func, output=None, options: dict = None): + if options is None: + options = {} + return self.binarySortedMapPartitionsWithIndex( + other=other, + binary_sorted_map_partitions_with_index_op=_lifted_join_merge_to_sbmpwi(func), + ) + + +def _lifted_map_to_io_serdes(_f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes): + def _lifted(_index, _iter): + for out_k, out_v in _f(_index, _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes)): + yield output_key_serdes.serialize(out_k), output_value_serdes.serialize(out_v) + + return _lifted + + +def _serdes_wrapped_generator(_iter, key_serdes, value_serdes): + for k, v in _iter: + yield key_serdes.deserialize(k), value_serdes.deserialize(v) + + +def _value_serdes_wrapped_generator(_iter, value_serdes): + for k, v in _iter: + yield k, value_serdes.deserialize(v) + + +def _lifted_mpwi_map_to_serdes(_f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes): + def _lifted(_index, _iter): + for out_k, out_v in _f(_index, _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes)): + yield output_key_serdes.serialize(out_k), output_value_serdes.serialize(out_v) + + return _lifted + + +def _lifted_mpwi_reduce_to_serdes(_f, output_value_serdes): + if _f is None: + return None + + def _lifted(x, y): + return output_value_serdes.serialize( + _f( + output_value_serdes.deserialize(x), + output_value_serdes.deserialize(y), + ) + ) + + return _lifted + + +def _lifted_map_values_to_mpwi(map_value_op: Callable[[Any], Any]): + def _lifted(_index, _iter): + for _k, _v in _iter: + yield _k, map_value_op(_v) + + return _lifted + + +def _lifted_map_to_mpwi(map_op: Callable[[Any, Any], Tuple[Any, Any]]): + def _lifted(_index, _iter): + for _k, _v in _iter: + yield map_op(_k, _v) + + return _lifted + + +def _lifted_map_reduce_partitions_to_mpwi(map_partition_op: Callable[[Iterable], Iterable]): + def _lifted(_index, _iter): + return map_partition_op(_iter) + + return _lifted + + +def _get_generator_with_last_key(_iter): + cache = [None] + + def _generator(): + for k, v in _iter: + cache[0] = k + yield k, v + + return _generator, cache + + +def _lifted_apply_partitions_to_mpwi(apply_partition_op: Callable[[Iterable], Any]): + def _lifted(_index, _iter): + _iter_set_cache, _cache = _get_generator_with_last_key(_iter) + value = apply_partition_op(_iter_set_cache()) + key = _cache[0] + if key is None: + return [] + return [(key, value)] + + return _lifted + + +def _lifted_map_partitions_to_mpwi(map_partition_op: Callable[[Iterable], Iterable]): + def _lifted(_index, _iter): + return map_partition_op(_iter) + + return _lifted + + +def _lifted_flat_map_to_mpwi(flat_map_op: Callable[[Any, Any], Iterable[Tuple[Any, Any]]]): + def _lifted(_index, _iter): + for _k, _v in _iter: + yield from flat_map_op(_k, _v) + + return _lifted + + +def _lifted_sample_to_mpwi(fraction, seed=None): + def _lifted(_index, _iter): + # TODO: should we use the same seed for all partitions? + random_state = random.Random(seed) + for _k, _v in _iter: + if random_state.random() < fraction: + yield _k, _v + + return _lifted + + +def _lifted_reduce_to_serdes(reduce_op, value_serdes): + def _lifted(x, y): + return value_serdes.serialize( + reduce_op( + value_serdes.deserialize(x), + value_serdes.deserialize(y), + ) + ) + + return _lifted + + +def _lifted_sorted_binary_map_partitions_with_index_to_serdes( + _f, left_value_serdes, right_value_serdes, output_value_serdes +): + def _lifted(_index, left_iter, right_iter): + for out_k_bytes, out_v in _f( + _index, + _value_serdes_wrapped_generator(left_iter, left_value_serdes), + _value_serdes_wrapped_generator(right_iter, right_value_serdes), + ): + yield out_k_bytes, output_value_serdes.serialize(out_v) + + return _lifted + + +def _lifted_join_merge_to_sbmpwi(join_merge_op): + def _lifted(_index, _left_iter, _right_iter): + return _merge_intersecting_keys(_left_iter, _right_iter, join_merge_op) + + return _lifted + + +def _lifted_union_merge_to_sbmpwi(join_merge_op): + def _lifted(_index, _left_iter, _right_iter): + return _merge_union_keys(_left_iter, _right_iter, join_merge_op) + + return _lifted + + +def _lifted_subtract_by_key_to_sbmpwi(): + def _lifted(_index, _left_iter, _right_iter): + return _subtract_by_key(_left_iter, _right_iter) + + return _lifted + + +def _merge_intersecting_keys(iter1, iter2, merge_op): + try: + item1 = next(iter1) + item2 = next(iter2) + except StopIteration: + return + + while True: + key1, value1 = item1 + key2, value2 = item2 + + if key1 == key2: + yield key1, merge_op(value1, value2) + try: + item1 = next(iter1) + item2 = next(iter2) + except StopIteration: + break + elif key1 < key2: + try: + item1 = next(iter1) + except StopIteration: + break + else: # key1 > key2 + try: + item2 = next(iter2) + except StopIteration: + break + + +def _merge_union_keys(iter1, iter2, merge_op): + try: + item1 = next(iter1) + except StopIteration: + item1 = None + + try: + item2 = next(iter2) + except StopIteration: + item2 = None + + if item1 is None and item2 is None: + return + + while item1 is not None and item2 is not None: + key1, value1 = item1 + key2, value2 = item2 + + if key1 == key2: + yield key1, merge_op(value1, value2) + try: + item1 = next(iter1) + except StopIteration: + item1 = None + try: + item2 = next(iter2) + except StopIteration: + item2 = None + elif key1 < key2: + yield key1, value1 + try: + item1 = next(iter1) + except StopIteration: + item1 = None + else: # key1 > key2 + yield key2, value2 + try: + item2 = next(iter2) + except StopIteration: + item2 = None + + if item1 is not None: + yield item1 + yield from iter1 + elif item2 is not None: + yield item2 + yield from iter2 + + +def _subtract_by_key(iter1, iter2): + try: + item1 = next(iter1) + except StopIteration: + return + + try: + item2 = next(iter2) + except StopIteration: + yield item1 + yield from iter1 + return + + while item1 is not None and item2 is not None: + key1, value1 = item1 + key2, value2 = item2 + + if key1 == key2: + try: + item1 = next(iter1) + except StopIteration: + item1 = None + try: + item2 = next(iter2) + except StopIteration: + item2 = None + elif key1 < key2: + yield item1 + try: + item1 = next(iter1) + except StopIteration: + item1 = None + else: # key1 > key2 + try: + item2 = next(iter2) + except StopIteration: + item2 = None + + if item1 is not None: + yield item1 + yield from iter1 From a019c8ae6e0921d0fc2432960a3fb630e852368e Mon Sep 17 00:00:00 2001 From: sagewe Date: Tue, 16 Jan 2024 11:57:00 +0800 Subject: [PATCH 03/20] feat: enable monkey patch and pass toy (#708) 1. impl rollsite wrap 2. add monkey patch to patch package import use custom package loader 3. refactor some class Signed-off-by: sagewe --- python/eggroll/backport/__init__.py | 26 ++ .../backport/_module_rollpair_patch.py | 19 ++ .../backport/_module_rollsite_patch.py | 19 ++ .../eggroll/backport/_module_session_patch.py | 19 ++ .../eggroll/backport/_package_loader_patch.py | 69 +++++ python/eggroll/backport/_partitioner.py | 21 ++ python/eggroll/backport/_serdes.py | 31 ++ .../backport/_wrap_rollpair.py} | 276 +++++++++++------- python/eggroll/backport/_wrap_rollsite.py | 98 +++++++ python/eggroll/backport/_wrap_session.py | 44 +++ .../eggroll/computing/roll_pair/_roll_pair.py | 7 +- python/eggroll/federation/__init__.py | 3 +- python/eggroll/federation/_rollsite.py | 20 ++ python/eggroll_2x/__init__.py | 3 - python/eggroll_2x/_partitioner.py | 5 - python/eggroll_2x/_serdes.py | 12 - 16 files changed, 541 insertions(+), 131 deletions(-) create mode 100644 python/eggroll/backport/__init__.py create mode 100644 python/eggroll/backport/_module_rollpair_patch.py create mode 100644 python/eggroll/backport/_module_rollsite_patch.py create mode 100644 python/eggroll/backport/_module_session_patch.py create mode 100644 python/eggroll/backport/_package_loader_patch.py create mode 100644 python/eggroll/backport/_partitioner.py create mode 100644 python/eggroll/backport/_serdes.py rename python/{eggroll_2x/_wrap.py => eggroll/backport/_wrap_rollpair.py} (70%) create mode 100644 python/eggroll/backport/_wrap_rollsite.py create mode 100644 python/eggroll/backport/_wrap_session.py delete mode 100644 python/eggroll_2x/__init__.py delete mode 100644 python/eggroll_2x/_partitioner.py delete mode 100644 python/eggroll_2x/_serdes.py diff --git a/python/eggroll/backport/__init__.py b/python/eggroll/backport/__init__.py new file mode 100644 index 000000000..5ee52e980 --- /dev/null +++ b/python/eggroll/backport/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +import sys + + +def backport_patch(): + from ._package_loader_patch import Eggroll2xPackageFinder + sys.meta_path.append(Eggroll2xPackageFinder()) + + +if __name__ == "__main__": + backport_patch() diff --git a/python/eggroll/backport/_module_rollpair_patch.py b/python/eggroll/backport/_module_rollpair_patch.py new file mode 100644 index 000000000..856f02056 --- /dev/null +++ b/python/eggroll/backport/_module_rollpair_patch.py @@ -0,0 +1,19 @@ +# Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +from ._wrap_rollpair import WrappedRp as RollPair, runtime_init + +__all__ = ["RollPair", "runtime_init"] diff --git a/python/eggroll/backport/_module_rollsite_patch.py b/python/eggroll/backport/_module_rollsite_patch.py new file mode 100644 index 000000000..ac1222fdb --- /dev/null +++ b/python/eggroll/backport/_module_rollsite_patch.py @@ -0,0 +1,19 @@ +# Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +from ._wrap_rollsite import WrappedRollSiteContext as RollSiteContext + +__all__ = ["RollSiteContext"] diff --git a/python/eggroll/backport/_module_session_patch.py b/python/eggroll/backport/_module_session_patch.py new file mode 100644 index 000000000..0de62ccdd --- /dev/null +++ b/python/eggroll/backport/_module_session_patch.py @@ -0,0 +1,19 @@ +# Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +from ._wrap_session import session_init + +__all__ = ["session_init"] diff --git a/python/eggroll/backport/_package_loader_patch.py b/python/eggroll/backport/_package_loader_patch.py new file mode 100644 index 000000000..9e9b7a7b0 --- /dev/null +++ b/python/eggroll/backport/_package_loader_patch.py @@ -0,0 +1,69 @@ +# Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +import importlib.abc +import importlib.machinery + + +class Eggroll2xPackageFinder(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path, target=None): + if ( + fullname.startswith("eggroll.core.session") + or fullname.startswith("eggroll.roll_pair") + or fullname.startswith("eggroll.roll_site") + ): + return importlib.machinery.ModuleSpec( + fullname, + Eggroll2xLoader(), + ) + + +class Eggroll2xLoader(importlib.abc.Loader): + def create_module(self, spec): + if spec.name == "eggroll.core.session": + from . import _module_session_patch + + return _module_session_patch + + if spec.name == "eggroll.roll_pair": + return _create_duck_package(spec.name) + + if spec.name == "eggroll.roll_pair.roll_pair": + from . import _module_rollpair_patch + + return _module_rollpair_patch + + if spec.name == "eggroll.roll_site": + return _create_duck_package(spec.name) + + if spec.name == "eggroll.roll_site.roll_site": + from . import _module_rollsite_patch + + return _module_rollsite_patch + + def exec_module(self, module): + pass + + +def _create_duck_package(name): + import types + import sys + + duck = types.ModuleType(name) + duck.__path__ = [] + duck.__package__ = name + sys.modules[name] = duck + return duck diff --git a/python/eggroll/backport/_partitioner.py b/python/eggroll/backport/_partitioner.py new file mode 100644 index 000000000..b7c5de72c --- /dev/null +++ b/python/eggroll/backport/_partitioner.py @@ -0,0 +1,21 @@ +# Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +import mmh3 + + +def mmh3_partitioner(key: bytes, total_partitions): + return mmh3.hash(key) % total_partitions diff --git a/python/eggroll/backport/_serdes.py b/python/eggroll/backport/_serdes.py new file mode 100644 index 000000000..50aa3e1b6 --- /dev/null +++ b/python/eggroll/backport/_serdes.py @@ -0,0 +1,31 @@ +# Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +from pickle import dumps as p_dumps +from pickle import loads as p_loads + + +class UnrestrictedSerdes: + @staticmethod + def serialize(obj) -> bytes: + return p_dumps(obj) + + @staticmethod + def deserialize(bytes) -> object: + return p_loads(bytes) + + +Serdes = UnrestrictedSerdes diff --git a/python/eggroll_2x/_wrap.py b/python/eggroll/backport/_wrap_rollpair.py similarity index 70% rename from python/eggroll_2x/_wrap.py rename to python/eggroll/backport/_wrap_rollpair.py index 8b2d415b9..ebdadb9de 100644 --- a/python/eggroll_2x/_wrap.py +++ b/python/eggroll/backport/_wrap_rollpair.py @@ -1,32 +1,37 @@ -import random -import uuid +# Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +import typing from typing import Callable, Iterable, Any, Tuple -from eggroll.computing import runtime_init as _runtime_init, RollPairContext, RollPair -from eggroll.computing.tasks.store import StoreTypes -from eggroll.session import session_init as _session_init from ._partitioner import mmh3_partitioner -from ._serdes import UnrestrictedSerdes +from ._serdes import Serdes - -def session_init(session_id, options) -> "WrappedSession": - _session = _session_init(session_id=session_id, options=options) - return WrappedSession(session=_session) +if typing.TYPE_CHECKING: + from ._wrap_session import WrappedSession + from eggroll.computing import RollPairContext, RollPair def runtime_init(session: "WrappedSession") -> "WrappedRpc": + from eggroll.computing import runtime_init as _runtime_init + rpc = _runtime_init(session=session._session) return WrappedRpc(rpc=rpc) -class WrappedSession: - def __init__(self, session): - self._session = session - - def get_session_id(self): - return self._session.get_session_id() - - class WrappedRpc: def __init__(self, rpc: "RollPairContext"): self._rpc = rpc @@ -36,10 +41,15 @@ def session_id(self): return self._rpc.session.get_session_id() def load(self, namespace, name, options): + from eggroll.computing.tasks.store import StoreTypes + store_type = options.get("store_type", StoreTypes.ROLLPAIR_LMDB) return self._rpc.load_rp(namespace=namespace, name=name, store_type=store_type) def parallelize(self, data, options: dict = None): + from eggroll.computing.tasks.store import StoreTypes + import uuid + if options is None: options = {} namespace = options.get("namespace", None) @@ -59,12 +69,12 @@ def parallelize(self, data, options: dict = None): # generate data if include_key: data = ( - (UnrestrictedSerdes.serialize(key), UnrestrictedSerdes.serialize(value)) + (Serdes.serialize(key), Serdes.serialize(value)) for key, value in data ) else: data = ( - (UnrestrictedSerdes.serialize(i), UnrestrictedSerdes.serialize(value)) + (Serdes.serialize(i), Serdes.serialize(value)) for i, value in enumerate(data) ) return WrappedRp( @@ -92,9 +102,15 @@ def kill(self): class WrappedRp: - def __init__(self, rp: RollPair): + def __init__(self, rp: "RollPair"): self._rp = rp + def get_namespace(self): + return self._rp.get_namespace() + + def get_name(self): + return self._rp.get_name() + def get_partitions(self): return self._rp.get_partitions() @@ -102,33 +118,33 @@ def get(self, k, options: dict = None): if options is None: options = {} value = self._rp.get( - k=UnrestrictedSerdes.serialize(k), partitioner=mmh3_partitioner + k=Serdes.serialize(k), partitioner=mmh3_partitioner ) if value is None: return None else: - return UnrestrictedSerdes.deserialize(value) + return Serdes.deserialize(value) def put(self, k, v, options: dict = None): if options is None: options = {} self._rp.put( - k=UnrestrictedSerdes.serialize(k), - v=UnrestrictedSerdes.serialize(v), + k=Serdes.serialize(k), + v=Serdes.serialize(v), partitioner=mmh3_partitioner, ) def delete(self, k, options: dict = None): if options is None: options = {} - self._rp.delete(k=UnrestrictedSerdes.serialize(k), partitioner=mmh3_partitioner) + self._rp.delete(k=Serdes.serialize(k), partitioner=mmh3_partitioner) def count(self): return self._rp.count() def get_all(self, limit=None, options: dict = None): for k, v in self._rp.get_all(limit=limit): - yield UnrestrictedSerdes.deserialize(k), UnrestrictedSerdes.deserialize(v) + yield Serdes.deserialize(k), Serdes.deserialize(v) def put_all(self, items, output=None, options: dict = None): if options is None: @@ -137,14 +153,14 @@ def put_all(self, items, output=None, options: dict = None): if include_key: self._rp.put_all( ( - (UnrestrictedSerdes.serialize(k), UnrestrictedSerdes.serialize(v)) + (Serdes.serialize(k), Serdes.serialize(v)) for k, v in items ), ) else: self._rp.put_all( ( - (UnrestrictedSerdes.serialize(i), UnrestrictedSerdes.serialize(v)) + (Serdes.serialize(i), Serdes.serialize(v)) for i, v in enumerate(items) ), ) @@ -152,10 +168,13 @@ def put_all(self, items, output=None, options: dict = None): def take(self, n: int, options: dict = None): keys_only = options.get("keys_only", False) if keys_only: - return [UnrestrictedSerdes.deserialize(k) for k in self._rp.take(n=n, options=options)] + return [ + Serdes.deserialize(k) + for k in self._rp.take(n=n, options=options) + ] else: return [ - (UnrestrictedSerdes.deserialize(k), UnrestrictedSerdes.deserialize(v)) + (Serdes.deserialize(k), Serdes.deserialize(v)) for k, v in self._rp.take(n=n, options=options) ] @@ -177,29 +196,41 @@ def _map_reduce_partitions_with_index( output_num_partitions=None, ): if not shuffle and reduce_partition_op is not None: - raise ValueError("when shuffle is False, it is not allowed to specify reduce_partition_op") + raise ValueError( + "when shuffle is False, it is not allowed to specify reduce_partition_op" + ) if output_num_partitions is None: output_num_partitions = self._rp.get_partitions() - return WrappedRp(self._rp.map_reduce_partitions_with_index( - map_partition_op=_lifted_mpwi_map_to_serdes( - map_partition_op, UnrestrictedSerdes, UnrestrictedSerdes, UnrestrictedSerdes, UnrestrictedSerdes, - ), - reduce_partition_op=_lifted_mpwi_reduce_to_serdes(reduce_partition_op, UnrestrictedSerdes), - shuffle=shuffle, - input_key_serdes=UnrestrictedSerdes, - input_key_serdes_type=0, - input_value_serdes=UnrestrictedSerdes, - input_value_serdes_type=0, - input_partitioner=mmh3_partitioner, - input_partitioner_type=0, - output_key_serdes=UnrestrictedSerdes, - output_key_serdes_type=0, - output_value_serdes=UnrestrictedSerdes, - output_value_serdes_type=0, - output_partitioner=mmh3_partitioner, - output_partitioner_type=0, - output_num_partitions=output_num_partitions, - )) + + mapper = _lifted_map_to_io_serdes( + map_partition_op, + Serdes, + Serdes, + Serdes, + Serdes, + ) + return WrappedRp( + self._rp.map_reduce_partitions_with_index( + map_partition_op=mapper, + reduce_partition_op=_lifted_mpwi_reduce_to_serdes( + reduce_partition_op, Serdes + ), + shuffle=shuffle, + input_key_serdes=Serdes, + input_key_serdes_type=0, + input_value_serdes=Serdes, + input_value_serdes_type=0, + input_partitioner=mmh3_partitioner, + input_partitioner_type=0, + output_key_serdes=Serdes, + output_key_serdes_type=0, + output_value_serdes=Serdes, + output_value_serdes_type=0, + output_partitioner=mmh3_partitioner, + output_partitioner_type=0, + output_num_partitions=output_num_partitions, + ) + ) def map_values(self, func, output=None, options: dict = None): if options is None: @@ -220,7 +251,7 @@ def map(self, func, output=None, options: dict = None): def map_partitions(self, func, reduce_op=None, output=None, options: dict = None): if options is None: options = {} - shuffle = options.get('shuffle', True) + shuffle = options.get("shuffle", True) return self._map_reduce_partitions_with_index( _lifted_map_partitions_to_mpwi(func), reduce_partition_op=reduce_op, @@ -238,7 +269,7 @@ def collapse_partitions(self, func, output=None, options: dict = None): def map_partitions_with_index(self, func, output=None, options: dict = None): if options is None: options = {} - shuffle = options.get('shuffle', True) + shuffle = options.get("shuffle", True) return self._map_reduce_partitions_with_index( map_partition_op=func, shuffle=shuffle, @@ -256,9 +287,13 @@ def reduce(self, func, output=None, options: dict = None): if options is None: options = {} - return self._rp.reduce( - func=_lifted_reduce_to_serdes(func, UnrestrictedSerdes), + reduced = self._rp.reduce( + func=_lifted_reduce_to_serdes(func, Serdes), ) + if reduced is None: + return None + else: + return Serdes.deserialize(reduced) def sample(self, fraction, seed=None, output=None, options: dict = None): if options is None: @@ -281,29 +316,29 @@ def num_partitions(self): return self._rp.get_partitions() def repartition(self, num_partitions) -> "WrappedRp": - if ( - self.num_partitions == num_partitions - ): + if self.num_partitions == num_partitions: return self - return WrappedRp(self._rp.map_reduce_partitions_with_index( - map_partition_op=lambda i, x: x, - reduce_partition_op=None, - shuffle=True, - input_key_serdes=UnrestrictedSerdes, - input_key_serdes_type=0, - input_value_serdes=UnrestrictedSerdes, - input_value_serdes_type=0, - input_partitioner=mmh3_partitioner, - input_partitioner_type=0, - output_key_serdes=UnrestrictedSerdes, - output_key_serdes_type=0, - output_value_serdes=UnrestrictedSerdes, - output_value_serdes_type=0, - output_partitioner=mmh3_partitioner, - output_partitioner_type=0, - output_num_partitions=num_partitions, - )) + return WrappedRp( + self._rp.map_reduce_partitions_with_index( + map_partition_op=lambda i, x: x, + reduce_partition_op=None, + shuffle=True, + input_key_serdes=Serdes, + input_key_serdes_type=0, + input_value_serdes=Serdes, + input_value_serdes_type=0, + input_partitioner=mmh3_partitioner, + input_partitioner_type=0, + output_key_serdes=Serdes, + output_key_serdes_type=0, + output_value_serdes=Serdes, + output_value_serdes_type=0, + output_partitioner=mmh3_partitioner, + output_partitioner_type=0, + output_num_partitions=num_partitions, + ) + ) def repartition_with(self, other: "WrappedRp") -> Tuple["WrappedRp", "WrappedRp"]: if self._rp.num_partitions == other._rp.num_partitions: @@ -316,30 +351,34 @@ def repartition_with(self, other: "WrappedRp") -> Tuple["WrappedRp", "WrappedRp" def binarySortedMapPartitionsWithIndex( self, other: "WrappedRp", - binary_sorted_map_partitions_with_index_op: Callable[[int, Iterable, Iterable], Iterable], + binary_sorted_map_partitions_with_index_op: Callable[ + [int, Iterable, Iterable], Iterable + ], ): first, second = self.repartition_with(other) # apply binary_sorted_map_partitions_with_index_op - return WrappedRp(first._rp.binary_sorted_map_partitions_with_index( - other=second._rp, - binary_map_partitions_with_index_op=_lifted_sorted_binary_map_partitions_with_index_to_serdes( - binary_sorted_map_partitions_with_index_op, - UnrestrictedSerdes, - UnrestrictedSerdes, - UnrestrictedSerdes, - ), - key_serdes=UnrestrictedSerdes, - key_serdes_type=0, - partitioner=mmh3_partitioner, - partitioner_type=0, - first_input_value_serdes=UnrestrictedSerdes, - first_input_value_serdes_type=0, - second_input_value_serdes=UnrestrictedSerdes, - second_input_value_serdes_type=0, - output_value_serdes=UnrestrictedSerdes, - output_value_serdes_type=0, - )) + return WrappedRp( + first._rp.binary_sorted_map_partitions_with_index( + other=second._rp, + binary_map_partitions_with_index_op=_lifted_sorted_binary_map_partitions_with_index_to_serdes( + binary_sorted_map_partitions_with_index_op, + Serdes, + Serdes, + Serdes, + ), + key_serdes=Serdes, + key_serdes_type=0, + partitioner=mmh3_partitioner, + partitioner_type=0, + first_input_value_serdes=Serdes, + first_input_value_serdes_type=0, + second_input_value_serdes=Serdes, + second_input_value_serdes_type=0, + output_value_serdes=Serdes, + output_value_serdes_type=0, + ) + ) def subtract_by_key(self, other, output=None, options: dict = None): if options is None: @@ -354,7 +393,9 @@ def union(self, other, func=lambda v1, v2: v1, output=None, options: dict = None options = {} return self.binarySortedMapPartitionsWithIndex( other=other, - binary_sorted_map_partitions_with_index_op=_lifted_union_merge_to_sbmpwi(func), + binary_sorted_map_partitions_with_index_op=_lifted_union_merge_to_sbmpwi( + func + ), ) def join(self, other, func, output=None, options: dict = None): @@ -362,14 +403,23 @@ def join(self, other, func, output=None, options: dict = None): options = {} return self.binarySortedMapPartitionsWithIndex( other=other, - binary_sorted_map_partitions_with_index_op=_lifted_join_merge_to_sbmpwi(func), + binary_sorted_map_partitions_with_index_op=_lifted_join_merge_to_sbmpwi( + func + ), ) -def _lifted_map_to_io_serdes(_f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes): +def _lifted_map_to_io_serdes( + _f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes +): def _lifted(_index, _iter): - for out_k, out_v in _f(_index, _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes)): - yield output_key_serdes.serialize(out_k), output_value_serdes.serialize(out_v) + for out_k, out_v in _f( + _index, + _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes), + ): + yield output_key_serdes.serialize(out_k), output_value_serdes.serialize( + out_v + ) return _lifted @@ -384,10 +434,17 @@ def _value_serdes_wrapped_generator(_iter, value_serdes): yield k, value_serdes.deserialize(v) -def _lifted_mpwi_map_to_serdes(_f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes): +def _lifted_mpwi_map_to_serdes( + _f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes +): def _lifted(_index, _iter): - for out_k, out_v in _f(_index, _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes)): - yield output_key_serdes.serialize(out_k), output_value_serdes.serialize(out_v) + for out_k, out_v in _f( + _index, + _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes), + ): + yield output_key_serdes.serialize(out_k), output_value_serdes.serialize( + out_v + ) return _lifted @@ -423,7 +480,9 @@ def _lifted(_index, _iter): return _lifted -def _lifted_map_reduce_partitions_to_mpwi(map_partition_op: Callable[[Iterable], Iterable]): +def _lifted_map_reduce_partitions_to_mpwi( + map_partition_op: Callable[[Iterable], Iterable] +): def _lifted(_index, _iter): return map_partition_op(_iter) @@ -460,7 +519,9 @@ def _lifted(_index, _iter): return _lifted -def _lifted_flat_map_to_mpwi(flat_map_op: Callable[[Any, Any], Iterable[Tuple[Any, Any]]]): +def _lifted_flat_map_to_mpwi( + flat_map_op: Callable[[Any, Any], Iterable[Tuple[Any, Any]]] +): def _lifted(_index, _iter): for _k, _v in _iter: yield from flat_map_op(_k, _v) @@ -470,6 +531,7 @@ def _lifted(_index, _iter): def _lifted_sample_to_mpwi(fraction, seed=None): def _lifted(_index, _iter): + import random # TODO: should we use the same seed for all partitions? random_state = random.Random(seed) for _k, _v in _iter: diff --git a/python/eggroll/backport/_wrap_rollsite.py b/python/eggroll/backport/_wrap_rollsite.py new file mode 100644 index 000000000..6fe5efdee --- /dev/null +++ b/python/eggroll/backport/_wrap_rollsite.py @@ -0,0 +1,98 @@ +# Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +import typing + +if typing.TYPE_CHECKING: + from eggroll.federation import RollSite + from ._wrap_rollpair import WrappedRpc + + +def _init_roll_site_context( + roll_site_session_id, rp_ctx: "WrappedRpc", options: dict = None +): + from eggroll.federation import RollSiteContext as _RollSiteContext + + if options is None: + raise ValueError("options is None") + assert "self_role" in options, "self_role not in options" + assert "self_party_id" in options, "self_party_id not in options" + assert "proxy_endpoint" in options, "proxy_endpoint not in options" + party = (options["self_role"], options["self_party_id"]) + proxy_endpoint = options["proxy_endpoint"] + proxy_endpoint_host, proxy_endpoint_port = proxy_endpoint.split(":") + proxy_endpoint_port = int(proxy_endpoint_port) + return _RollSiteContext( + roll_site_session_id=roll_site_session_id, + rp_ctx=rp_ctx._rpc, + party=party, + proxy_endpoint_host=proxy_endpoint_host, + proxy_endpoint_port=proxy_endpoint_port, + options=options, + ) + + +class WrappedRollSiteContext: + def __init__( + self, roll_site_session_id, rp_ctx: "WrappedRpc", options: dict = None + ): + if options is None: + options = {} + self._options = options + self._rsc = _init_roll_site_context(roll_site_session_id, rp_ctx, options) + + def load(self, name, tag, options: dict = None): + if options is None: + options = {} + final_options = self._options.copy() + final_options.update(options) + from eggroll.federation import RollSite + + return WrappedRollSite( + RollSite(name=name, tag=tag, rs_ctx=self._rsc, options=final_options) + ) + + +class WrappedRollSite: + def __init__(self, roll_site: "RollSite"): + self._roll_site = roll_site + + def push(self, obj, parties: list = None, options: dict = None): + from ._wrap_rollpair import WrappedRp + from ._serdes import Serdes + + if isinstance(obj, WrappedRp): + return self._roll_site.push_rp(obj._rp, parties, options) + else: + return self._roll_site.push_bytes(Serdes.serialize(obj), parties, options) + + def pull(self, parties: list = None, options: dict = None): + return self._roll_site.pull_with_lift(lifter=_lifter, parties=parties) + + +def _lifter(func): + def _inner(*args, **kwargs): + from ._wrap_rollpair import WrappedRp + from ._serdes import Serdes + from eggroll.computing import RollPair + + rp_or_obj = func(*args, **kwargs) + if isinstance(rp_or_obj, RollPair): + return WrappedRp(rp_or_obj) + else: + return Serdes.deserialize(rp_or_obj) + + return _inner diff --git a/python/eggroll/backport/_wrap_session.py b/python/eggroll/backport/_wrap_session.py new file mode 100644 index 000000000..271a37fda --- /dev/null +++ b/python/eggroll/backport/_wrap_session.py @@ -0,0 +1,44 @@ +# Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +import typing + +if typing.TYPE_CHECKING: + from eggroll.session import ErSession + + +def session_init(session_id, options) -> "WrappedSession": + import os + from eggroll.session import session_init as _session_init + + assert "EGGROLL_HOME" in os.environ, "EGGROLL_HOME not set" + config_properties_file = os.path.join( + os.environ["EGGROLL_HOME"], "conf", "eggroll.properties" + ) + _session = _session_init( + session_id=session_id, + options=options, + config_properties_file=config_properties_file, + ) + return WrappedSession(session=_session) + + +class WrappedSession: + def __init__(self, session: "ErSession"): + self._session = session + + def get_session_id(self): + return self._session.get_session_id() diff --git a/python/eggroll/computing/roll_pair/_roll_pair.py b/python/eggroll/computing/roll_pair/_roll_pair.py index 5e1e35b52..00d351cfd 100644 --- a/python/eggroll/computing/roll_pair/_roll_pair.py +++ b/python/eggroll/computing/roll_pair/_roll_pair.py @@ -35,9 +35,10 @@ class RollPair(object): def __getstate__(self): - raise NotImplementedError( - "pickling RollPair is not expected behavior, if you really need it, please create an issue at" - ) + return None + # raise NotImplementedError( + # "pickling RollPair is not expected behavior, if you really need it, please create an issue at" + # ) def __init__(self, er_store: ErStore, rp_ctx: "RollPairContext", gc_enabled=None): self._ctx = rp_ctx diff --git a/python/eggroll/federation/__init__.py b/python/eggroll/federation/__init__.py index 421c0f0b4..259342c4f 100644 --- a/python/eggroll/federation/__init__.py +++ b/python/eggroll/federation/__init__.py @@ -16,5 +16,6 @@ from ._rollsite_context import RollSiteContext +from ._rollsite import RollSite -__all__ = ["RollSiteContext"] +__all__ = ["RollSiteContext", "RollSite"] diff --git a/python/eggroll/federation/_rollsite.py b/python/eggroll/federation/_rollsite.py index ca7217747..0de4637a0 100644 --- a/python/eggroll/federation/_rollsite.py +++ b/python/eggroll/federation/_rollsite.py @@ -110,3 +110,23 @@ def pull(self, parties: list = None): ) ) return futures + + def pull_with_lift(self, lifter: typing.Callable[[typing.Callable], typing.Callable], parties: list = None): + futures = [] + for src_role, src_party_id in parties: + src_party_id = str(src_party_id) + rs_header = ErRollSiteHeader( + roll_site_session_id=self.roll_site_session_id, + name=self.name, + tag=self.tag, + src_role=src_role, + src_party_id=src_party_id, + dst_role=self.local_role, + dst_party_id=self.party_id, + ) + futures.append( + self._receive_executor_pool.submit( + lifter(self._impl_instance._pull_one), rs_header + ) + ) + return futures diff --git a/python/eggroll_2x/__init__.py b/python/eggroll_2x/__init__.py deleted file mode 100644 index 60da1f253..000000000 --- a/python/eggroll_2x/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from ._wrap import session_init, runtime_init - -__all__ = ["session_init", "runtime_init"] diff --git a/python/eggroll_2x/_partitioner.py b/python/eggroll_2x/_partitioner.py deleted file mode 100644 index 17aaf1f70..000000000 --- a/python/eggroll_2x/_partitioner.py +++ /dev/null @@ -1,5 +0,0 @@ -import mmh3 - - -def mmh3_partitioner(key: bytes, total_partitions): - return mmh3.hash(key) % total_partitions diff --git a/python/eggroll_2x/_serdes.py b/python/eggroll_2x/_serdes.py deleted file mode 100644 index 00846174a..000000000 --- a/python/eggroll_2x/_serdes.py +++ /dev/null @@ -1,12 +0,0 @@ -from pickle import dumps as p_dumps -from pickle import loads as p_loads - - -class UnrestrictedSerdes: - @staticmethod - def serialize(obj) -> bytes: - return p_dumps(obj) - - @staticmethod - def deserialize(bytes) -> object: - return p_loads(bytes) From df5dd58991eeb41cfdeda472145543dd30f84106 Mon Sep 17 00:00:00 2001 From: sagewe Date: Tue, 16 Jan 2024 13:13:51 +0800 Subject: [PATCH 04/20] fix: proto backport (#708) since eggroll 2.x use protobuf version is 3.x while eggroll 3.x use protobuf version 4.x, we need a way to import packages generated with correct version. since we introduce custom package loader before, we can achieve goals by 1. regenerate proto files with protobuf 3.x to backport directory 2. insert custom package loader to the meta_path head to make sure import of 'eggroll.core.proto' can be hijacked Signed-off-by: sagewe --- python/eggroll/backport/__init__.py | 3 +- .../eggroll/backport/_package_loader_patch.py | 12 +- python/eggroll/backport/_wrap_rollpair.py | 62 +- python/eggroll/backport/_wrap_rollsite.py | 4 +- python/eggroll/backport/proto/__init__.py | 20 + .../eggroll/backport/proto/basic_meta_pb2.py | 147 +++ .../eggroll/backport/proto/basic_meta_pb2.pyi | 305 +++++++ python/eggroll/backport/proto/command_pb2.py | 60 ++ python/eggroll/backport/proto/command_pb2.pyi | 104 +++ .../backport/proto/command_pb2_grpc.py | 99 +++ .../eggroll/backport/proto/containers_pb2.py | 198 +++++ .../eggroll/backport/proto/containers_pb2.pyi | 360 ++++++++ .../backport/proto/deepspeed_download_pb2.py | 80 ++ .../backport/proto/deepspeed_download_pb2.pyi | 129 +++ .../proto/deepspeed_download_pb2_grpc.py | 99 +++ .../eggroll/backport/proto/deepspeed_pb2.py | 372 ++++++++ .../eggroll/backport/proto/deepspeed_pb2.pyi | 621 +++++++++++++ .../backport/proto/deepspeed_pb2_grpc.py | 264 ++++++ python/eggroll/backport/proto/egg_pb2.py | 154 ++++ python/eggroll/backport/proto/egg_pb2.pyi | 251 ++++++ python/eggroll/backport/proto/extend_pb2.py | 47 + python/eggroll/backport/proto/extend_pb2.pyi | 80 ++ .../eggroll/backport/proto/extend_pb2_grpc.py | 69 ++ python/eggroll/backport/proto/meta_pb2.py | 402 +++++++++ python/eggroll/backport/proto/meta_pb2.pyi | 840 ++++++++++++++++++ python/eggroll/backport/proto/proxy_pb2.py | 142 +++ python/eggroll/backport/proto/proxy_pb2.pyi | 326 +++++++ .../eggroll/backport/proto/proxy_pb2_grpc.py | 230 +++++ python/eggroll/backport/proto/transfer_pb2.py | 152 ++++ .../eggroll/backport/proto/transfer_pb2.pyi | 312 +++++++ .../backport/proto/transfer_pb2_grpc.py | 135 +++ 31 files changed, 6038 insertions(+), 41 deletions(-) create mode 100644 python/eggroll/backport/proto/__init__.py create mode 100644 python/eggroll/backport/proto/basic_meta_pb2.py create mode 100644 python/eggroll/backport/proto/basic_meta_pb2.pyi create mode 100644 python/eggroll/backport/proto/command_pb2.py create mode 100644 python/eggroll/backport/proto/command_pb2.pyi create mode 100644 python/eggroll/backport/proto/command_pb2_grpc.py create mode 100644 python/eggroll/backport/proto/containers_pb2.py create mode 100644 python/eggroll/backport/proto/containers_pb2.pyi create mode 100644 python/eggroll/backport/proto/deepspeed_download_pb2.py create mode 100644 python/eggroll/backport/proto/deepspeed_download_pb2.pyi create mode 100644 python/eggroll/backport/proto/deepspeed_download_pb2_grpc.py create mode 100644 python/eggroll/backport/proto/deepspeed_pb2.py create mode 100644 python/eggroll/backport/proto/deepspeed_pb2.pyi create mode 100644 python/eggroll/backport/proto/deepspeed_pb2_grpc.py create mode 100644 python/eggroll/backport/proto/egg_pb2.py create mode 100644 python/eggroll/backport/proto/egg_pb2.pyi create mode 100644 python/eggroll/backport/proto/extend_pb2.py create mode 100644 python/eggroll/backport/proto/extend_pb2.pyi create mode 100644 python/eggroll/backport/proto/extend_pb2_grpc.py create mode 100644 python/eggroll/backport/proto/meta_pb2.py create mode 100644 python/eggroll/backport/proto/meta_pb2.pyi create mode 100644 python/eggroll/backport/proto/proxy_pb2.py create mode 100644 python/eggroll/backport/proto/proxy_pb2.pyi create mode 100644 python/eggroll/backport/proto/proxy_pb2_grpc.py create mode 100644 python/eggroll/backport/proto/transfer_pb2.py create mode 100644 python/eggroll/backport/proto/transfer_pb2.pyi create mode 100644 python/eggroll/backport/proto/transfer_pb2_grpc.py diff --git a/python/eggroll/backport/__init__.py b/python/eggroll/backport/__init__.py index 5ee52e980..6817c1bca 100644 --- a/python/eggroll/backport/__init__.py +++ b/python/eggroll/backport/__init__.py @@ -19,7 +19,8 @@ def backport_patch(): from ._package_loader_patch import Eggroll2xPackageFinder - sys.meta_path.append(Eggroll2xPackageFinder()) + + sys.meta_path.insert(0, Eggroll2xPackageFinder()) if __name__ == "__main__": diff --git a/python/eggroll/backport/_package_loader_patch.py b/python/eggroll/backport/_package_loader_patch.py index 9e9b7a7b0..ef08363ad 100644 --- a/python/eggroll/backport/_package_loader_patch.py +++ b/python/eggroll/backport/_package_loader_patch.py @@ -21,9 +21,10 @@ class Eggroll2xPackageFinder(importlib.abc.MetaPathFinder): def find_spec(self, fullname, path, target=None): if ( - fullname.startswith("eggroll.core.session") - or fullname.startswith("eggroll.roll_pair") - or fullname.startswith("eggroll.roll_site") + fullname.startswith("eggroll.core.session") + or fullname.startswith("eggroll.roll_pair") + or fullname.startswith("eggroll.roll_site") + or fullname == "eggroll.core.proto" ): return importlib.machinery.ModuleSpec( fullname, @@ -33,6 +34,11 @@ def find_spec(self, fullname, path, target=None): class Eggroll2xLoader(importlib.abc.Loader): def create_module(self, spec): + if spec.name == "eggroll.core.proto": + from . import proto + + return proto + if spec.name == "eggroll.core.session": from . import _module_session_patch diff --git a/python/eggroll/backport/_wrap_rollpair.py b/python/eggroll/backport/_wrap_rollpair.py index ebdadb9de..58b4640ac 100644 --- a/python/eggroll/backport/_wrap_rollpair.py +++ b/python/eggroll/backport/_wrap_rollpair.py @@ -69,8 +69,7 @@ def parallelize(self, data, options: dict = None): # generate data if include_key: data = ( - (Serdes.serialize(key), Serdes.serialize(value)) - for key, value in data + (Serdes.serialize(key), Serdes.serialize(value)) for key, value in data ) else: data = ( @@ -117,9 +116,7 @@ def get_partitions(self): def get(self, k, options: dict = None): if options is None: options = {} - value = self._rp.get( - k=Serdes.serialize(k), partitioner=mmh3_partitioner - ) + value = self._rp.get(k=Serdes.serialize(k), partitioner=mmh3_partitioner) if value is None: return None else: @@ -152,10 +149,7 @@ def put_all(self, items, output=None, options: dict = None): include_key = options.get("include_key", True) if include_key: self._rp.put_all( - ( - (Serdes.serialize(k), Serdes.serialize(v)) - for k, v in items - ), + ((Serdes.serialize(k), Serdes.serialize(v)) for k, v in items), ) else: self._rp.put_all( @@ -168,10 +162,7 @@ def put_all(self, items, output=None, options: dict = None): def take(self, n: int, options: dict = None): keys_only = options.get("keys_only", False) if keys_only: - return [ - Serdes.deserialize(k) - for k in self._rp.take(n=n, options=options) - ] + return [Serdes.deserialize(k) for k in self._rp.take(n=n, options=options)] else: return [ (Serdes.deserialize(k), Serdes.deserialize(v)) @@ -189,11 +180,11 @@ def destroy(self, options: dict = None): self._rp.destroy() def _map_reduce_partitions_with_index( - self, - map_partition_op: Callable[[int, Iterable], Iterable], - reduce_partition_op: Callable[[Any, Any], Any] = None, - shuffle=True, - output_num_partitions=None, + self, + map_partition_op: Callable[[int, Iterable], Iterable], + reduce_partition_op: Callable[[Any, Any], Any] = None, + shuffle=True, + output_num_partitions=None, ): if not shuffle and reduce_partition_op is not None: raise ValueError( @@ -349,11 +340,11 @@ def repartition_with(self, other: "WrappedRp") -> Tuple["WrappedRp", "WrappedRp" return self.repartition(other.num_partitions), other def binarySortedMapPartitionsWithIndex( - self, - other: "WrappedRp", - binary_sorted_map_partitions_with_index_op: Callable[ - [int, Iterable, Iterable], Iterable - ], + self, + other: "WrappedRp", + binary_sorted_map_partitions_with_index_op: Callable[ + [int, Iterable, Iterable], Iterable + ], ): first, second = self.repartition_with(other) @@ -410,12 +401,12 @@ def join(self, other, func, output=None, options: dict = None): def _lifted_map_to_io_serdes( - _f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes + _f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes ): def _lifted(_index, _iter): for out_k, out_v in _f( - _index, - _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes), + _index, + _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes), ): yield output_key_serdes.serialize(out_k), output_value_serdes.serialize( out_v @@ -435,12 +426,12 @@ def _value_serdes_wrapped_generator(_iter, value_serdes): def _lifted_mpwi_map_to_serdes( - _f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes + _f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes ): def _lifted(_index, _iter): for out_k, out_v in _f( - _index, - _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes), + _index, + _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes), ): yield output_key_serdes.serialize(out_k), output_value_serdes.serialize( out_v @@ -481,7 +472,7 @@ def _lifted(_index, _iter): def _lifted_map_reduce_partitions_to_mpwi( - map_partition_op: Callable[[Iterable], Iterable] + map_partition_op: Callable[[Iterable], Iterable] ): def _lifted(_index, _iter): return map_partition_op(_iter) @@ -520,7 +511,7 @@ def _lifted(_index, _iter): def _lifted_flat_map_to_mpwi( - flat_map_op: Callable[[Any, Any], Iterable[Tuple[Any, Any]]] + flat_map_op: Callable[[Any, Any], Iterable[Tuple[Any, Any]]] ): def _lifted(_index, _iter): for _k, _v in _iter: @@ -532,6 +523,7 @@ def _lifted(_index, _iter): def _lifted_sample_to_mpwi(fraction, seed=None): def _lifted(_index, _iter): import random + # TODO: should we use the same seed for all partitions? random_state = random.Random(seed) for _k, _v in _iter: @@ -554,13 +546,13 @@ def _lifted(x, y): def _lifted_sorted_binary_map_partitions_with_index_to_serdes( - _f, left_value_serdes, right_value_serdes, output_value_serdes + _f, left_value_serdes, right_value_serdes, output_value_serdes ): def _lifted(_index, left_iter, right_iter): for out_k_bytes, out_v in _f( - _index, - _value_serdes_wrapped_generator(left_iter, left_value_serdes), - _value_serdes_wrapped_generator(right_iter, right_value_serdes), + _index, + _value_serdes_wrapped_generator(left_iter, left_value_serdes), + _value_serdes_wrapped_generator(right_iter, right_value_serdes), ): yield out_k_bytes, output_value_serdes.serialize(out_v) diff --git a/python/eggroll/backport/_wrap_rollsite.py b/python/eggroll/backport/_wrap_rollsite.py index 6fe5efdee..31590629d 100644 --- a/python/eggroll/backport/_wrap_rollsite.py +++ b/python/eggroll/backport/_wrap_rollsite.py @@ -22,7 +22,7 @@ def _init_roll_site_context( - roll_site_session_id, rp_ctx: "WrappedRpc", options: dict = None + roll_site_session_id, rp_ctx: "WrappedRpc", options: dict = None ): from eggroll.federation import RollSiteContext as _RollSiteContext @@ -47,7 +47,7 @@ def _init_roll_site_context( class WrappedRollSiteContext: def __init__( - self, roll_site_session_id, rp_ctx: "WrappedRpc", options: dict = None + self, roll_site_session_id, rp_ctx: "WrappedRpc", options: dict = None ): if options is None: options = {} diff --git a/python/eggroll/backport/proto/__init__.py b/python/eggroll/backport/proto/__init__.py new file mode 100644 index 000000000..5a179018f --- /dev/null +++ b/python/eggroll/backport/proto/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright 2019 The Eggroll Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +import os + +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) \ No newline at end of file diff --git a/python/eggroll/backport/proto/basic_meta_pb2.py b/python/eggroll/backport/proto/basic_meta_pb2.py new file mode 100644 index 000000000..6c2d589cf --- /dev/null +++ b/python/eggroll/backport/proto/basic_meta_pb2.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: basic-meta.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x10\x62\x61sic-meta.proto\x12\x1e\x63om.webank.ai.eggroll.api.core\"6\n\x08\x45ndpoint\x12\n\n\x02ip\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\x12\x10\n\x08hostname\x18\x03 \x01(\t\"H\n\tEndpoints\x12;\n\tendpoints\x18\x01 \x03(\x0b\x32(.com.webank.ai.eggroll.api.core.Endpoint\"H\n\x04\x44\x61ta\x12\x0e\n\x06isNull\x18\x01 \x01(\x08\x12\x14\n\x0chostLanguage\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c\"F\n\x0cRepeatedData\x12\x36\n\x08\x64\x61talist\x18\x01 \x03(\x0b\x32$.com.webank.ai.eggroll.api.core.Data\"u\n\x0b\x43\x61llRequest\x12\x0f\n\x07isAsync\x18\x01 \x01(\x08\x12\x0f\n\x07timeout\x18\x02 \x01(\x03\x12\x0f\n\x07\x63ommand\x18\x03 \x01(\t\x12\x33\n\x05param\x18\x04 \x01(\x0b\x32$.com.webank.ai.eggroll.api.core.Data\"\x88\x01\n\x0c\x43\x61llResponse\x12\x42\n\x0creturnStatus\x18\x01 \x01(\x0b\x32,.com.webank.ai.eggroll.api.core.ReturnStatus\x12\x34\n\x06result\x18\x02 \x01(\x0b\x32$.com.webank.ai.eggroll.api.core.Data\"\"\n\x03Job\x12\r\n\x05jobId\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"Y\n\x04Task\x12\x30\n\x03job\x18\x01 \x01(\x0b\x32#.com.webank.ai.eggroll.api.core.Job\x12\x0e\n\x06taskId\x18\x02 \x01(\x03\x12\x0f\n\x07tableId\x18\x03 \x01(\x03\"N\n\x06Result\x12\x32\n\x04task\x18\x01 \x01(\x0b\x32$.com.webank.ai.eggroll.api.core.Task\x12\x10\n\x08resultId\x18\x02 \x01(\x03\"-\n\x0cReturnStatus\x12\x0c\n\x04\x63ode\x18\x01 \x01(\x05\x12\x0f\n\x07message\x18\x02 \x01(\t\"\xe2\x01\n\x0bSessionInfo\x12\x11\n\tsessionId\x18\x01 \x01(\t\x12\x61\n\x13\x63omputingEngineConf\x18\x02 \x03(\x0b\x32\x44.com.webank.ai.eggroll.api.core.SessionInfo.ComputingEngineConfEntry\x12\x14\n\x0cnamingPolicy\x18\x03 \x01(\t\x12\x0b\n\x03tag\x18\x04 \x01(\t\x1a:\n\x18\x43omputingEngineConfEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x62\x06proto3') + + + +_ENDPOINT = DESCRIPTOR.message_types_by_name['Endpoint'] +_ENDPOINTS = DESCRIPTOR.message_types_by_name['Endpoints'] +_DATA = DESCRIPTOR.message_types_by_name['Data'] +_REPEATEDDATA = DESCRIPTOR.message_types_by_name['RepeatedData'] +_CALLREQUEST = DESCRIPTOR.message_types_by_name['CallRequest'] +_CALLRESPONSE = DESCRIPTOR.message_types_by_name['CallResponse'] +_JOB = DESCRIPTOR.message_types_by_name['Job'] +_TASK = DESCRIPTOR.message_types_by_name['Task'] +_RESULT = DESCRIPTOR.message_types_by_name['Result'] +_RETURNSTATUS = DESCRIPTOR.message_types_by_name['ReturnStatus'] +_SESSIONINFO = DESCRIPTOR.message_types_by_name['SessionInfo'] +_SESSIONINFO_COMPUTINGENGINECONFENTRY = _SESSIONINFO.nested_types_by_name['ComputingEngineConfEntry'] +Endpoint = _reflection.GeneratedProtocolMessageType('Endpoint', (_message.Message,), { + 'DESCRIPTOR' : _ENDPOINT, + '__module__' : 'basic_meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Endpoint) + }) +_sym_db.RegisterMessage(Endpoint) + +Endpoints = _reflection.GeneratedProtocolMessageType('Endpoints', (_message.Message,), { + 'DESCRIPTOR' : _ENDPOINTS, + '__module__' : 'basic_meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Endpoints) + }) +_sym_db.RegisterMessage(Endpoints) + +Data = _reflection.GeneratedProtocolMessageType('Data', (_message.Message,), { + 'DESCRIPTOR' : _DATA, + '__module__' : 'basic_meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Data) + }) +_sym_db.RegisterMessage(Data) + +RepeatedData = _reflection.GeneratedProtocolMessageType('RepeatedData', (_message.Message,), { + 'DESCRIPTOR' : _REPEATEDDATA, + '__module__' : 'basic_meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.RepeatedData) + }) +_sym_db.RegisterMessage(RepeatedData) + +CallRequest = _reflection.GeneratedProtocolMessageType('CallRequest', (_message.Message,), { + 'DESCRIPTOR' : _CALLREQUEST, + '__module__' : 'basic_meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.CallRequest) + }) +_sym_db.RegisterMessage(CallRequest) + +CallResponse = _reflection.GeneratedProtocolMessageType('CallResponse', (_message.Message,), { + 'DESCRIPTOR' : _CALLRESPONSE, + '__module__' : 'basic_meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.CallResponse) + }) +_sym_db.RegisterMessage(CallResponse) + +Job = _reflection.GeneratedProtocolMessageType('Job', (_message.Message,), { + 'DESCRIPTOR' : _JOB, + '__module__' : 'basic_meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Job) + }) +_sym_db.RegisterMessage(Job) + +Task = _reflection.GeneratedProtocolMessageType('Task', (_message.Message,), { + 'DESCRIPTOR' : _TASK, + '__module__' : 'basic_meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Task) + }) +_sym_db.RegisterMessage(Task) + +Result = _reflection.GeneratedProtocolMessageType('Result', (_message.Message,), { + 'DESCRIPTOR' : _RESULT, + '__module__' : 'basic_meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.Result) + }) +_sym_db.RegisterMessage(Result) + +ReturnStatus = _reflection.GeneratedProtocolMessageType('ReturnStatus', (_message.Message,), { + 'DESCRIPTOR' : _RETURNSTATUS, + '__module__' : 'basic_meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.ReturnStatus) + }) +_sym_db.RegisterMessage(ReturnStatus) + +SessionInfo = _reflection.GeneratedProtocolMessageType('SessionInfo', (_message.Message,), { + + 'ComputingEngineConfEntry' : _reflection.GeneratedProtocolMessageType('ComputingEngineConfEntry', (_message.Message,), { + 'DESCRIPTOR' : _SESSIONINFO_COMPUTINGENGINECONFENTRY, + '__module__' : 'basic_meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.SessionInfo.ComputingEngineConfEntry) + }) + , + 'DESCRIPTOR' : _SESSIONINFO, + '__module__' : 'basic_meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.core.SessionInfo) + }) +_sym_db.RegisterMessage(SessionInfo) +_sym_db.RegisterMessage(SessionInfo.ComputingEngineConfEntry) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _SESSIONINFO_COMPUTINGENGINECONFENTRY._options = None + _SESSIONINFO_COMPUTINGENGINECONFENTRY._serialized_options = b'8\001' + _ENDPOINT._serialized_start=52 + _ENDPOINT._serialized_end=106 + _ENDPOINTS._serialized_start=108 + _ENDPOINTS._serialized_end=180 + _DATA._serialized_start=182 + _DATA._serialized_end=254 + _REPEATEDDATA._serialized_start=256 + _REPEATEDDATA._serialized_end=326 + _CALLREQUEST._serialized_start=328 + _CALLREQUEST._serialized_end=445 + _CALLRESPONSE._serialized_start=448 + _CALLRESPONSE._serialized_end=584 + _JOB._serialized_start=586 + _JOB._serialized_end=620 + _TASK._serialized_start=622 + _TASK._serialized_end=711 + _RESULT._serialized_start=713 + _RESULT._serialized_end=791 + _RETURNSTATUS._serialized_start=793 + _RETURNSTATUS._serialized_end=838 + _SESSIONINFO._serialized_start=841 + _SESSIONINFO._serialized_end=1067 + _SESSIONINFO_COMPUTINGENGINECONFENTRY._serialized_start=1009 + _SESSIONINFO_COMPUTINGENGINECONFENTRY._serialized_end=1067 +# @@protoc_insertion_point(module_scope) diff --git a/python/eggroll/backport/proto/basic_meta_pb2.pyi b/python/eggroll/backport/proto/basic_meta_pb2.pyi new file mode 100644 index 000000000..dae6a6ca4 --- /dev/null +++ b/python/eggroll/backport/proto/basic_meta_pb2.pyi @@ -0,0 +1,305 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file + +Copyright 2019 The Eggroll Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import collections.abc +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.message +import sys + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing_extensions.final +class Endpoint(google.protobuf.message.Message): + """network endpoint""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + IP_FIELD_NUMBER: builtins.int + PORT_FIELD_NUMBER: builtins.int + HOSTNAME_FIELD_NUMBER: builtins.int + ip: builtins.str + """ip address""" + port: builtins.int + """port""" + hostname: builtins.str + """host name""" + def __init__( + self, + *, + ip: builtins.str = ..., + port: builtins.int = ..., + hostname: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["hostname", b"hostname", "ip", b"ip", "port", b"port"]) -> None: ... + +global___Endpoint = Endpoint + +@typing_extensions.final +class Endpoints(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ENDPOINTS_FIELD_NUMBER: builtins.int + @property + def endpoints(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Endpoint]: ... + def __init__( + self, + *, + endpoints: collections.abc.Iterable[global___Endpoint] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["endpoints", b"endpoints"]) -> None: ... + +global___Endpoints = Endpoints + +@typing_extensions.final +class Data(google.protobuf.message.Message): + """general container for serialized data""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ISNULL_FIELD_NUMBER: builtins.int + HOSTLANGUAGE_FIELD_NUMBER: builtins.int + TYPE_FIELD_NUMBER: builtins.int + DATA_FIELD_NUMBER: builtins.int + isNull: builtins.bool + """whether the data is actually 'null' (e.g. null in Java, None in python, NULL in c/c++""" + hostLanguage: builtins.str + """the host language which serializes it""" + type: builtins.str + """data type in host language""" + data: builtins.bytes + """actual data serialized in bytes""" + def __init__( + self, + *, + isNull: builtins.bool = ..., + hostLanguage: builtins.str = ..., + type: builtins.str = ..., + data: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["data", b"data", "hostLanguage", b"hostLanguage", "isNull", b"isNull", "type", b"type"]) -> None: ... + +global___Data = Data + +@typing_extensions.final +class RepeatedData(google.protobuf.message.Message): + """general container for data list""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DATALIST_FIELD_NUMBER: builtins.int + @property + def datalist(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Data]: + """list of data""" + def __init__( + self, + *, + datalist: collections.abc.Iterable[global___Data] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["datalist", b"datalist"]) -> None: ... + +global___RepeatedData = RepeatedData + +@typing_extensions.final +class CallRequest(google.protobuf.message.Message): + """general message for a call request""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ISASYNC_FIELD_NUMBER: builtins.int + TIMEOUT_FIELD_NUMBER: builtins.int + COMMAND_FIELD_NUMBER: builtins.int + PARAM_FIELD_NUMBER: builtins.int + isAsync: builtins.bool + """whether the call is async. ignored in phase 1""" + timeout: builtins.int + """in ms""" + command: builtins.str + """call command. ignored in phase 1""" + @property + def param(self) -> global___Data: + """call input param""" + def __init__( + self, + *, + isAsync: builtins.bool = ..., + timeout: builtins.int = ..., + command: builtins.str = ..., + param: global___Data | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["param", b"param"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["command", b"command", "isAsync", b"isAsync", "param", b"param", "timeout", b"timeout"]) -> None: ... + +global___CallRequest = CallRequest + +@typing_extensions.final +class CallResponse(google.protobuf.message.Message): + """general message for a call response + todo: merge call response with Return Status + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RETURNSTATUS_FIELD_NUMBER: builtins.int + RESULT_FIELD_NUMBER: builtins.int + @property + def returnStatus(self) -> global___ReturnStatus: + """return status""" + @property + def result(self) -> global___Data: + """call result""" + def __init__( + self, + *, + returnStatus: global___ReturnStatus | None = ..., + result: global___Data | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["result", b"result", "returnStatus", b"returnStatus"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["result", b"result", "returnStatus", b"returnStatus"]) -> None: ... + +global___CallResponse = CallResponse + +@typing_extensions.final +class Job(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + JOBID_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + jobId: builtins.str + name: builtins.str + def __init__( + self, + *, + jobId: builtins.str = ..., + name: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["jobId", b"jobId", "name", b"name"]) -> None: ... + +global___Job = Job + +@typing_extensions.final +class Task(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + JOB_FIELD_NUMBER: builtins.int + TASKID_FIELD_NUMBER: builtins.int + TABLEID_FIELD_NUMBER: builtins.int + @property + def job(self) -> global___Job: ... + taskId: builtins.int + tableId: builtins.int + def __init__( + self, + *, + job: global___Job | None = ..., + taskId: builtins.int = ..., + tableId: builtins.int = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["job", b"job"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["job", b"job", "tableId", b"tableId", "taskId", b"taskId"]) -> None: ... + +global___Task = Task + +@typing_extensions.final +class Result(google.protobuf.message.Message): + """reserved for driver async call result""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TASK_FIELD_NUMBER: builtins.int + RESULTID_FIELD_NUMBER: builtins.int + @property + def task(self) -> global___Task: ... + resultId: builtins.int + def __init__( + self, + *, + task: global___Task | None = ..., + resultId: builtins.int = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["task", b"task"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["resultId", b"resultId", "task", b"task"]) -> None: ... + +global___Result = Result + +@typing_extensions.final +class ReturnStatus(google.protobuf.message.Message): + """generic return status""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CODE_FIELD_NUMBER: builtins.int + MESSAGE_FIELD_NUMBER: builtins.int + code: builtins.int + message: builtins.str + def __init__( + self, + *, + code: builtins.int = ..., + message: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["code", b"code", "message", b"message"]) -> None: ... + +global___ReturnStatus = ReturnStatus + +@typing_extensions.final +class SessionInfo(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class ComputingEngineConfEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + SESSIONID_FIELD_NUMBER: builtins.int + COMPUTINGENGINECONF_FIELD_NUMBER: builtins.int + NAMINGPOLICY_FIELD_NUMBER: builtins.int + TAG_FIELD_NUMBER: builtins.int + sessionId: builtins.str + @property + def computingEngineConf(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ... + namingPolicy: builtins.str + tag: builtins.str + def __init__( + self, + *, + sessionId: builtins.str = ..., + computingEngineConf: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + namingPolicy: builtins.str = ..., + tag: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["computingEngineConf", b"computingEngineConf", "namingPolicy", b"namingPolicy", "sessionId", b"sessionId", "tag", b"tag"]) -> None: ... + +global___SessionInfo = SessionInfo diff --git a/python/eggroll/backport/proto/command_pb2.py b/python/eggroll/backport/proto/command_pb2.py new file mode 100644 index 000000000..c550b6bf1 --- /dev/null +++ b/python/eggroll/backport/proto/command_pb2.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: command.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rcommand.proto\x12\x1f\x63om.webank.eggroll.core.command\"\xb3\x01\n\x0e\x43ommandRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x0c\n\x04\x61rgs\x18\x03 \x03(\x0c\x12K\n\x06kwargs\x18\x04 \x03(\x0b\x32;.com.webank.eggroll.core.command.CommandRequest.KwargsEntry\x1a-\n\x0bKwargsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\"\x8b\x01\n\x0f\x43ommandResponse\x12\n\n\x02id\x18\x01 \x01(\t\x12@\n\x07request\x18\x02 \x01(\x0b\x32/.com.webank.eggroll.core.command.CommandRequest\x12\x0f\n\x07results\x18\x03 \x03(\x0c\x12\x0c\n\x04\x63ode\x18\x04 \x01(\t\x12\x0b\n\x03msg\x18\x05 \x01(\t2\xf0\x01\n\x0e\x43ommandService\x12i\n\x04\x63\x61ll\x12/.com.webank.eggroll.core.command.CommandRequest\x1a\x30.com.webank.eggroll.core.command.CommandResponse\x12s\n\ncallStream\x12/.com.webank.eggroll.core.command.CommandRequest\x1a\x30.com.webank.eggroll.core.command.CommandResponse(\x01\x30\x01\x62\x06proto3') + + + +_COMMANDREQUEST = DESCRIPTOR.message_types_by_name['CommandRequest'] +_COMMANDREQUEST_KWARGSENTRY = _COMMANDREQUEST.nested_types_by_name['KwargsEntry'] +_COMMANDRESPONSE = DESCRIPTOR.message_types_by_name['CommandResponse'] +CommandRequest = _reflection.GeneratedProtocolMessageType('CommandRequest', (_message.Message,), { + + 'KwargsEntry' : _reflection.GeneratedProtocolMessageType('KwargsEntry', (_message.Message,), { + 'DESCRIPTOR' : _COMMANDREQUEST_KWARGSENTRY, + '__module__' : 'command_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.command.CommandRequest.KwargsEntry) + }) + , + 'DESCRIPTOR' : _COMMANDREQUEST, + '__module__' : 'command_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.command.CommandRequest) + }) +_sym_db.RegisterMessage(CommandRequest) +_sym_db.RegisterMessage(CommandRequest.KwargsEntry) + +CommandResponse = _reflection.GeneratedProtocolMessageType('CommandResponse', (_message.Message,), { + 'DESCRIPTOR' : _COMMANDRESPONSE, + '__module__' : 'command_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.command.CommandResponse) + }) +_sym_db.RegisterMessage(CommandResponse) + +_COMMANDSERVICE = DESCRIPTOR.services_by_name['CommandService'] +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _COMMANDREQUEST_KWARGSENTRY._options = None + _COMMANDREQUEST_KWARGSENTRY._serialized_options = b'8\001' + _COMMANDREQUEST._serialized_start=51 + _COMMANDREQUEST._serialized_end=230 + _COMMANDREQUEST_KWARGSENTRY._serialized_start=185 + _COMMANDREQUEST_KWARGSENTRY._serialized_end=230 + _COMMANDRESPONSE._serialized_start=233 + _COMMANDRESPONSE._serialized_end=372 + _COMMANDSERVICE._serialized_start=375 + _COMMANDSERVICE._serialized_end=615 +# @@protoc_insertion_point(module_scope) diff --git a/python/eggroll/backport/proto/command_pb2.pyi b/python/eggroll/backport/proto/command_pb2.pyi new file mode 100644 index 000000000..1d0d2bfff --- /dev/null +++ b/python/eggroll/backport/proto/command_pb2.pyi @@ -0,0 +1,104 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file + +Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import collections.abc +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.message +import sys + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing_extensions.final +class CommandRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class KwargsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.bytes + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + ID_FIELD_NUMBER: builtins.int + URI_FIELD_NUMBER: builtins.int + ARGS_FIELD_NUMBER: builtins.int + KWARGS_FIELD_NUMBER: builtins.int + id: builtins.str + uri: builtins.str + @property + def args(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.bytes]: ... + @property + def kwargs(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.bytes]: + """reserved for scala / python etc.""" + def __init__( + self, + *, + id: builtins.str = ..., + uri: builtins.str = ..., + args: collections.abc.Iterable[builtins.bytes] | None = ..., + kwargs: collections.abc.Mapping[builtins.str, builtins.bytes] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["args", b"args", "id", b"id", "kwargs", b"kwargs", "uri", b"uri"]) -> None: ... + +global___CommandRequest = CommandRequest + +@typing_extensions.final +class CommandResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ID_FIELD_NUMBER: builtins.int + REQUEST_FIELD_NUMBER: builtins.int + RESULTS_FIELD_NUMBER: builtins.int + CODE_FIELD_NUMBER: builtins.int + MSG_FIELD_NUMBER: builtins.int + id: builtins.str + @property + def request(self) -> global___CommandRequest: ... + @property + def results(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.bytes]: ... + code: builtins.str + msg: builtins.str + def __init__( + self, + *, + id: builtins.str = ..., + request: global___CommandRequest | None = ..., + results: collections.abc.Iterable[builtins.bytes] | None = ..., + code: builtins.str = ..., + msg: builtins.str = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["request", b"request"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["code", b"code", "id", b"id", "msg", b"msg", "request", b"request", "results", b"results"]) -> None: ... + +global___CommandResponse = CommandResponse diff --git a/python/eggroll/backport/proto/command_pb2_grpc.py b/python/eggroll/backport/proto/command_pb2_grpc.py new file mode 100644 index 000000000..8adc09caa --- /dev/null +++ b/python/eggroll/backport/proto/command_pb2_grpc.py @@ -0,0 +1,99 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +import command_pb2 as command__pb2 + + +class CommandServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.call = channel.unary_unary( + '/com.webank.eggroll.core.command.CommandService/call', + request_serializer=command__pb2.CommandRequest.SerializeToString, + response_deserializer=command__pb2.CommandResponse.FromString, + ) + self.callStream = channel.stream_stream( + '/com.webank.eggroll.core.command.CommandService/callStream', + request_serializer=command__pb2.CommandRequest.SerializeToString, + response_deserializer=command__pb2.CommandResponse.FromString, + ) + + +class CommandServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def call(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def callStream(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_CommandServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'call': grpc.unary_unary_rpc_method_handler( + servicer.call, + request_deserializer=command__pb2.CommandRequest.FromString, + response_serializer=command__pb2.CommandResponse.SerializeToString, + ), + 'callStream': grpc.stream_stream_rpc_method_handler( + servicer.callStream, + request_deserializer=command__pb2.CommandRequest.FromString, + response_serializer=command__pb2.CommandResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'com.webank.eggroll.core.command.CommandService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class CommandService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def call(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/com.webank.eggroll.core.command.CommandService/call', + command__pb2.CommandRequest.SerializeToString, + command__pb2.CommandResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def callStream(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_stream(request_iterator, target, '/com.webank.eggroll.core.command.CommandService/callStream', + command__pb2.CommandRequest.SerializeToString, + command__pb2.CommandResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/python/eggroll/backport/proto/containers_pb2.py b/python/eggroll/backport/proto/containers_pb2.py new file mode 100644 index 000000000..1a487356a --- /dev/null +++ b/python/eggroll/backport/proto/containers_pb2.py @@ -0,0 +1,198 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: containers.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x10\x63ontainers.proto\x12\x1c\x63om.webank.eggroll.core.meta\"\xca\x06\n\x16StartContainersRequest\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x10\n\x08job_type\x18\x03 \x01(\t\x12\x19\n\x11\x63ommand_arguments\x18\x04 \x03(\t\x12m\n\x15\x65nvironment_variables\x18\x05 \x03(\x0b\x32N.com.webank.eggroll.core.meta.StartContainersRequest.EnvironmentVariablesEntry\x12N\n\x05\x66iles\x18\x06 \x03(\x0b\x32?.com.webank.eggroll.core.meta.StartContainersRequest.FilesEntry\x12[\n\x0czipped_files\x18\x07 \x03(\x0b\x32\x45.com.webank.eggroll.core.meta.StartContainersRequest.ZippedFilesEntry\x12h\n\x13typed_extra_configs\x18\x08 \x03(\x0b\x32K.com.webank.eggroll.core.meta.StartContainersRequest.TypedExtraConfigsEntry\x12R\n\x07options\x18\t \x03(\x0b\x32\x41.com.webank.eggroll.core.meta.StartContainersRequest.OptionsEntry\x1a;\n\x19\x45nvironmentVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a,\n\nFilesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a\x32\n\x10ZippedFilesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a\x38\n\x16TypedExtraConfigsEntry\x12\x0b\n\x03key\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xf9\x01\n\x18\x44\x65\x65pspeedContainerConfig\x12\x1c\n\x14\x63uda_visible_devices\x18\x02 \x03(\r\x12\x12\n\nworld_size\x18\x03 \x01(\r\x12\x12\n\ncross_rank\x18\x04 \x01(\r\x12\x12\n\ncross_size\x18\x05 \x01(\r\x12\x12\n\nlocal_size\x18\x06 \x01(\r\x12\x12\n\nlocal_rank\x18\x07 \x01(\r\x12\x0c\n\x04rank\x18\x08 \x01(\r\x12\x12\n\nstore_host\x18\t \x01(\t\x12\x12\n\nstore_port\x18\n \x01(\x05\x12\x14\n\x0cstore_prefix\x18\x0b \x01(\t\x12\x0f\n\x07\x62\x61\x63kend\x18\x0c \x01(\t\"-\n\x17StartContainersResponse\x12\x12\n\nsession_id\x18\x01 \x01(\t\"B\n\x15StopContainersRequest\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\x15\n\rcontainer_ids\x18\x02 \x03(\x03\",\n\x16StopContainersResponse\x12\x12\n\nsession_id\x18\x01 \x01(\t\"B\n\x15KillContainersRequest\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\x15\n\rcontainer_ids\x18\x02 \x03(\x03\",\n\x16KillContainersResponse\x12\x12\n\nsession_id\x18\x01 \x01(\t\"\xb8\x01\n\x19\x44ownloadContainersRequest\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\x15\n\rcontainer_ids\x18\x02 \x03(\x03\x12\x17\n\x0f\x63ompress_method\x18\x03 \x01(\t\x12\x16\n\x0e\x63ompress_level\x18\x04 \x01(\x05\x12?\n\x0c\x63ontent_type\x18\x05 \x01(\x0e\x32).com.webank.eggroll.core.meta.ContentType\"{\n\x1a\x44ownloadContainersResponse\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12I\n\x11\x63ontainer_content\x18\x02 \x03(\x0b\x32..com.webank.eggroll.core.meta.ContainerContent\"R\n\x10\x43ontainerContent\x12\x14\n\x0c\x63ontainer_id\x18\x01 \x01(\x03\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\x0c\x12\x17\n\x0f\x63ompress_method\x18\x03 \x01(\t*8\n\x0b\x43ontentType\x12\x07\n\x03\x41LL\x10\x00\x12\n\n\x06MODELS\x10\x01\x12\x08\n\x04LOGS\x10\x02\x12\n\n\x06RESULT\x10\x03\x62\x06proto3') + +_CONTENTTYPE = DESCRIPTOR.enum_types_by_name['ContentType'] +ContentType = enum_type_wrapper.EnumTypeWrapper(_CONTENTTYPE) +ALL = 0 +MODELS = 1 +LOGS = 2 +RESULT = 3 + + +_STARTCONTAINERSREQUEST = DESCRIPTOR.message_types_by_name['StartContainersRequest'] +_STARTCONTAINERSREQUEST_ENVIRONMENTVARIABLESENTRY = _STARTCONTAINERSREQUEST.nested_types_by_name['EnvironmentVariablesEntry'] +_STARTCONTAINERSREQUEST_FILESENTRY = _STARTCONTAINERSREQUEST.nested_types_by_name['FilesEntry'] +_STARTCONTAINERSREQUEST_ZIPPEDFILESENTRY = _STARTCONTAINERSREQUEST.nested_types_by_name['ZippedFilesEntry'] +_STARTCONTAINERSREQUEST_TYPEDEXTRACONFIGSENTRY = _STARTCONTAINERSREQUEST.nested_types_by_name['TypedExtraConfigsEntry'] +_STARTCONTAINERSREQUEST_OPTIONSENTRY = _STARTCONTAINERSREQUEST.nested_types_by_name['OptionsEntry'] +_DEEPSPEEDCONTAINERCONFIG = DESCRIPTOR.message_types_by_name['DeepspeedContainerConfig'] +_STARTCONTAINERSRESPONSE = DESCRIPTOR.message_types_by_name['StartContainersResponse'] +_STOPCONTAINERSREQUEST = DESCRIPTOR.message_types_by_name['StopContainersRequest'] +_STOPCONTAINERSRESPONSE = DESCRIPTOR.message_types_by_name['StopContainersResponse'] +_KILLCONTAINERSREQUEST = DESCRIPTOR.message_types_by_name['KillContainersRequest'] +_KILLCONTAINERSRESPONSE = DESCRIPTOR.message_types_by_name['KillContainersResponse'] +_DOWNLOADCONTAINERSREQUEST = DESCRIPTOR.message_types_by_name['DownloadContainersRequest'] +_DOWNLOADCONTAINERSRESPONSE = DESCRIPTOR.message_types_by_name['DownloadContainersResponse'] +_CONTAINERCONTENT = DESCRIPTOR.message_types_by_name['ContainerContent'] +StartContainersRequest = _reflection.GeneratedProtocolMessageType('StartContainersRequest', (_message.Message,), { + + 'EnvironmentVariablesEntry' : _reflection.GeneratedProtocolMessageType('EnvironmentVariablesEntry', (_message.Message,), { + 'DESCRIPTOR' : _STARTCONTAINERSREQUEST_ENVIRONMENTVARIABLESENTRY, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StartContainersRequest.EnvironmentVariablesEntry) + }) + , + + 'FilesEntry' : _reflection.GeneratedProtocolMessageType('FilesEntry', (_message.Message,), { + 'DESCRIPTOR' : _STARTCONTAINERSREQUEST_FILESENTRY, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StartContainersRequest.FilesEntry) + }) + , + + 'ZippedFilesEntry' : _reflection.GeneratedProtocolMessageType('ZippedFilesEntry', (_message.Message,), { + 'DESCRIPTOR' : _STARTCONTAINERSREQUEST_ZIPPEDFILESENTRY, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StartContainersRequest.ZippedFilesEntry) + }) + , + + 'TypedExtraConfigsEntry' : _reflection.GeneratedProtocolMessageType('TypedExtraConfigsEntry', (_message.Message,), { + 'DESCRIPTOR' : _STARTCONTAINERSREQUEST_TYPEDEXTRACONFIGSENTRY, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StartContainersRequest.TypedExtraConfigsEntry) + }) + , + + 'OptionsEntry' : _reflection.GeneratedProtocolMessageType('OptionsEntry', (_message.Message,), { + 'DESCRIPTOR' : _STARTCONTAINERSREQUEST_OPTIONSENTRY, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StartContainersRequest.OptionsEntry) + }) + , + 'DESCRIPTOR' : _STARTCONTAINERSREQUEST, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StartContainersRequest) + }) +_sym_db.RegisterMessage(StartContainersRequest) +_sym_db.RegisterMessage(StartContainersRequest.EnvironmentVariablesEntry) +_sym_db.RegisterMessage(StartContainersRequest.FilesEntry) +_sym_db.RegisterMessage(StartContainersRequest.ZippedFilesEntry) +_sym_db.RegisterMessage(StartContainersRequest.TypedExtraConfigsEntry) +_sym_db.RegisterMessage(StartContainersRequest.OptionsEntry) + +DeepspeedContainerConfig = _reflection.GeneratedProtocolMessageType('DeepspeedContainerConfig', (_message.Message,), { + 'DESCRIPTOR' : _DEEPSPEEDCONTAINERCONFIG, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.DeepspeedContainerConfig) + }) +_sym_db.RegisterMessage(DeepspeedContainerConfig) + +StartContainersResponse = _reflection.GeneratedProtocolMessageType('StartContainersResponse', (_message.Message,), { + 'DESCRIPTOR' : _STARTCONTAINERSRESPONSE, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StartContainersResponse) + }) +_sym_db.RegisterMessage(StartContainersResponse) + +StopContainersRequest = _reflection.GeneratedProtocolMessageType('StopContainersRequest', (_message.Message,), { + 'DESCRIPTOR' : _STOPCONTAINERSREQUEST, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StopContainersRequest) + }) +_sym_db.RegisterMessage(StopContainersRequest) + +StopContainersResponse = _reflection.GeneratedProtocolMessageType('StopContainersResponse', (_message.Message,), { + 'DESCRIPTOR' : _STOPCONTAINERSRESPONSE, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StopContainersResponse) + }) +_sym_db.RegisterMessage(StopContainersResponse) + +KillContainersRequest = _reflection.GeneratedProtocolMessageType('KillContainersRequest', (_message.Message,), { + 'DESCRIPTOR' : _KILLCONTAINERSREQUEST, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.KillContainersRequest) + }) +_sym_db.RegisterMessage(KillContainersRequest) + +KillContainersResponse = _reflection.GeneratedProtocolMessageType('KillContainersResponse', (_message.Message,), { + 'DESCRIPTOR' : _KILLCONTAINERSRESPONSE, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.KillContainersResponse) + }) +_sym_db.RegisterMessage(KillContainersResponse) + +DownloadContainersRequest = _reflection.GeneratedProtocolMessageType('DownloadContainersRequest', (_message.Message,), { + 'DESCRIPTOR' : _DOWNLOADCONTAINERSREQUEST, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.DownloadContainersRequest) + }) +_sym_db.RegisterMessage(DownloadContainersRequest) + +DownloadContainersResponse = _reflection.GeneratedProtocolMessageType('DownloadContainersResponse', (_message.Message,), { + 'DESCRIPTOR' : _DOWNLOADCONTAINERSRESPONSE, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.DownloadContainersResponse) + }) +_sym_db.RegisterMessage(DownloadContainersResponse) + +ContainerContent = _reflection.GeneratedProtocolMessageType('ContainerContent', (_message.Message,), { + 'DESCRIPTOR' : _CONTAINERCONTENT, + '__module__' : 'containers_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.ContainerContent) + }) +_sym_db.RegisterMessage(ContainerContent) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _STARTCONTAINERSREQUEST_ENVIRONMENTVARIABLESENTRY._options = None + _STARTCONTAINERSREQUEST_ENVIRONMENTVARIABLESENTRY._serialized_options = b'8\001' + _STARTCONTAINERSREQUEST_FILESENTRY._options = None + _STARTCONTAINERSREQUEST_FILESENTRY._serialized_options = b'8\001' + _STARTCONTAINERSREQUEST_ZIPPEDFILESENTRY._options = None + _STARTCONTAINERSREQUEST_ZIPPEDFILESENTRY._serialized_options = b'8\001' + _STARTCONTAINERSREQUEST_TYPEDEXTRACONFIGSENTRY._options = None + _STARTCONTAINERSREQUEST_TYPEDEXTRACONFIGSENTRY._serialized_options = b'8\001' + _STARTCONTAINERSREQUEST_OPTIONSENTRY._options = None + _STARTCONTAINERSREQUEST_OPTIONSENTRY._serialized_options = b'8\001' + _CONTENTTYPE._serialized_start=1818 + _CONTENTTYPE._serialized_end=1874 + _STARTCONTAINERSREQUEST._serialized_start=51 + _STARTCONTAINERSREQUEST._serialized_end=893 + _STARTCONTAINERSREQUEST_ENVIRONMENTVARIABLESENTRY._serialized_start=630 + _STARTCONTAINERSREQUEST_ENVIRONMENTVARIABLESENTRY._serialized_end=689 + _STARTCONTAINERSREQUEST_FILESENTRY._serialized_start=691 + _STARTCONTAINERSREQUEST_FILESENTRY._serialized_end=735 + _STARTCONTAINERSREQUEST_ZIPPEDFILESENTRY._serialized_start=737 + _STARTCONTAINERSREQUEST_ZIPPEDFILESENTRY._serialized_end=787 + _STARTCONTAINERSREQUEST_TYPEDEXTRACONFIGSENTRY._serialized_start=789 + _STARTCONTAINERSREQUEST_TYPEDEXTRACONFIGSENTRY._serialized_end=845 + _STARTCONTAINERSREQUEST_OPTIONSENTRY._serialized_start=847 + _STARTCONTAINERSREQUEST_OPTIONSENTRY._serialized_end=893 + _DEEPSPEEDCONTAINERCONFIG._serialized_start=896 + _DEEPSPEEDCONTAINERCONFIG._serialized_end=1145 + _STARTCONTAINERSRESPONSE._serialized_start=1147 + _STARTCONTAINERSRESPONSE._serialized_end=1192 + _STOPCONTAINERSREQUEST._serialized_start=1194 + _STOPCONTAINERSREQUEST._serialized_end=1260 + _STOPCONTAINERSRESPONSE._serialized_start=1262 + _STOPCONTAINERSRESPONSE._serialized_end=1306 + _KILLCONTAINERSREQUEST._serialized_start=1308 + _KILLCONTAINERSREQUEST._serialized_end=1374 + _KILLCONTAINERSRESPONSE._serialized_start=1376 + _KILLCONTAINERSRESPONSE._serialized_end=1420 + _DOWNLOADCONTAINERSREQUEST._serialized_start=1423 + _DOWNLOADCONTAINERSREQUEST._serialized_end=1607 + _DOWNLOADCONTAINERSRESPONSE._serialized_start=1609 + _DOWNLOADCONTAINERSRESPONSE._serialized_end=1732 + _CONTAINERCONTENT._serialized_start=1734 + _CONTAINERCONTENT._serialized_end=1816 +# @@protoc_insertion_point(module_scope) diff --git a/python/eggroll/backport/proto/containers_pb2.pyi b/python/eggroll/backport/proto/containers_pb2.pyi new file mode 100644 index 000000000..ba79edb7f --- /dev/null +++ b/python/eggroll/backport/proto/containers_pb2.pyi @@ -0,0 +1,360 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" +import builtins +import collections.abc +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message +import sys +import typing + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _ContentType: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _ContentTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_ContentType.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + ALL: _ContentType.ValueType # 0 + MODELS: _ContentType.ValueType # 1 + LOGS: _ContentType.ValueType # 2 + RESULT: _ContentType.ValueType # 3 + +class ContentType(_ContentType, metaclass=_ContentTypeEnumTypeWrapper): ... + +ALL: ContentType.ValueType # 0 +MODELS: ContentType.ValueType # 1 +LOGS: ContentType.ValueType # 2 +RESULT: ContentType.ValueType # 3 +global___ContentType = ContentType + +@typing_extensions.final +class StartContainersRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class EnvironmentVariablesEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + @typing_extensions.final + class FilesEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.bytes + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + @typing_extensions.final + class ZippedFilesEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.bytes + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + @typing_extensions.final + class TypedExtraConfigsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.int + value: builtins.bytes + def __init__( + self, + *, + key: builtins.int = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + @typing_extensions.final + class OptionsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + SESSION_ID_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + JOB_TYPE_FIELD_NUMBER: builtins.int + COMMAND_ARGUMENTS_FIELD_NUMBER: builtins.int + ENVIRONMENT_VARIABLES_FIELD_NUMBER: builtins.int + FILES_FIELD_NUMBER: builtins.int + ZIPPED_FILES_FIELD_NUMBER: builtins.int + TYPED_EXTRA_CONFIGS_FIELD_NUMBER: builtins.int + OPTIONS_FIELD_NUMBER: builtins.int + session_id: builtins.str + name: builtins.str + job_type: builtins.str + @property + def command_arguments(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ... + @property + def environment_variables(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ... + @property + def files(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.bytes]: ... + @property + def zipped_files(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.bytes]: ... + @property + def typed_extra_configs(self) -> google.protobuf.internal.containers.ScalarMap[builtins.int, builtins.bytes]: ... + @property + def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ... + def __init__( + self, + *, + session_id: builtins.str = ..., + name: builtins.str = ..., + job_type: builtins.str = ..., + command_arguments: collections.abc.Iterable[builtins.str] | None = ..., + environment_variables: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + files: collections.abc.Mapping[builtins.str, builtins.bytes] | None = ..., + zipped_files: collections.abc.Mapping[builtins.str, builtins.bytes] | None = ..., + typed_extra_configs: collections.abc.Mapping[builtins.int, builtins.bytes] | None = ..., + options: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["command_arguments", b"command_arguments", "environment_variables", b"environment_variables", "files", b"files", "job_type", b"job_type", "name", b"name", "options", b"options", "session_id", b"session_id", "typed_extra_configs", b"typed_extra_configs", "zipped_files", b"zipped_files"]) -> None: ... + +global___StartContainersRequest = StartContainersRequest + +@typing_extensions.final +class DeepspeedContainerConfig(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CUDA_VISIBLE_DEVICES_FIELD_NUMBER: builtins.int + WORLD_SIZE_FIELD_NUMBER: builtins.int + CROSS_RANK_FIELD_NUMBER: builtins.int + CROSS_SIZE_FIELD_NUMBER: builtins.int + LOCAL_SIZE_FIELD_NUMBER: builtins.int + LOCAL_RANK_FIELD_NUMBER: builtins.int + RANK_FIELD_NUMBER: builtins.int + STORE_HOST_FIELD_NUMBER: builtins.int + STORE_PORT_FIELD_NUMBER: builtins.int + STORE_PREFIX_FIELD_NUMBER: builtins.int + BACKEND_FIELD_NUMBER: builtins.int + @property + def cuda_visible_devices(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ... + world_size: builtins.int + cross_rank: builtins.int + cross_size: builtins.int + local_size: builtins.int + local_rank: builtins.int + rank: builtins.int + store_host: builtins.str + store_port: builtins.int + store_prefix: builtins.str + backend: builtins.str + def __init__( + self, + *, + cuda_visible_devices: collections.abc.Iterable[builtins.int] | None = ..., + world_size: builtins.int = ..., + cross_rank: builtins.int = ..., + cross_size: builtins.int = ..., + local_size: builtins.int = ..., + local_rank: builtins.int = ..., + rank: builtins.int = ..., + store_host: builtins.str = ..., + store_port: builtins.int = ..., + store_prefix: builtins.str = ..., + backend: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["backend", b"backend", "cross_rank", b"cross_rank", "cross_size", b"cross_size", "cuda_visible_devices", b"cuda_visible_devices", "local_rank", b"local_rank", "local_size", b"local_size", "rank", b"rank", "store_host", b"store_host", "store_port", b"store_port", "store_prefix", b"store_prefix", "world_size", b"world_size"]) -> None: ... + +global___DeepspeedContainerConfig = DeepspeedContainerConfig + +@typing_extensions.final +class StartContainersResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + session_id: builtins.str + def __init__( + self, + *, + session_id: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["session_id", b"session_id"]) -> None: ... + +global___StartContainersResponse = StartContainersResponse + +@typing_extensions.final +class StopContainersRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + CONTAINER_IDS_FIELD_NUMBER: builtins.int + session_id: builtins.str + @property + def container_ids(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ... + def __init__( + self, + *, + session_id: builtins.str = ..., + container_ids: collections.abc.Iterable[builtins.int] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["container_ids", b"container_ids", "session_id", b"session_id"]) -> None: ... + +global___StopContainersRequest = StopContainersRequest + +@typing_extensions.final +class StopContainersResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + session_id: builtins.str + def __init__( + self, + *, + session_id: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["session_id", b"session_id"]) -> None: ... + +global___StopContainersResponse = StopContainersResponse + +@typing_extensions.final +class KillContainersRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + CONTAINER_IDS_FIELD_NUMBER: builtins.int + session_id: builtins.str + @property + def container_ids(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ... + def __init__( + self, + *, + session_id: builtins.str = ..., + container_ids: collections.abc.Iterable[builtins.int] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["container_ids", b"container_ids", "session_id", b"session_id"]) -> None: ... + +global___KillContainersRequest = KillContainersRequest + +@typing_extensions.final +class KillContainersResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + session_id: builtins.str + def __init__( + self, + *, + session_id: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["session_id", b"session_id"]) -> None: ... + +global___KillContainersResponse = KillContainersResponse + +@typing_extensions.final +class DownloadContainersRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + CONTAINER_IDS_FIELD_NUMBER: builtins.int + COMPRESS_METHOD_FIELD_NUMBER: builtins.int + COMPRESS_LEVEL_FIELD_NUMBER: builtins.int + CONTENT_TYPE_FIELD_NUMBER: builtins.int + session_id: builtins.str + @property + def container_ids(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ... + compress_method: builtins.str + compress_level: builtins.int + content_type: global___ContentType.ValueType + def __init__( + self, + *, + session_id: builtins.str = ..., + container_ids: collections.abc.Iterable[builtins.int] | None = ..., + compress_method: builtins.str = ..., + compress_level: builtins.int = ..., + content_type: global___ContentType.ValueType = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["compress_level", b"compress_level", "compress_method", b"compress_method", "container_ids", b"container_ids", "content_type", b"content_type", "session_id", b"session_id"]) -> None: ... + +global___DownloadContainersRequest = DownloadContainersRequest + +@typing_extensions.final +class DownloadContainersResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + CONTAINER_CONTENT_FIELD_NUMBER: builtins.int + session_id: builtins.str + @property + def container_content(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ContainerContent]: ... + def __init__( + self, + *, + session_id: builtins.str = ..., + container_content: collections.abc.Iterable[global___ContainerContent] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["container_content", b"container_content", "session_id", b"session_id"]) -> None: ... + +global___DownloadContainersResponse = DownloadContainersResponse + +@typing_extensions.final +class ContainerContent(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CONTAINER_ID_FIELD_NUMBER: builtins.int + CONTENT_FIELD_NUMBER: builtins.int + COMPRESS_METHOD_FIELD_NUMBER: builtins.int + container_id: builtins.int + content: builtins.bytes + compress_method: builtins.str + def __init__( + self, + *, + container_id: builtins.int = ..., + content: builtins.bytes = ..., + compress_method: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["compress_method", b"compress_method", "container_id", b"container_id", "content", b"content"]) -> None: ... + +global___ContainerContent = ContainerContent diff --git a/python/eggroll/backport/proto/deepspeed_download_pb2.py b/python/eggroll/backport/proto/deepspeed_download_pb2.py new file mode 100644 index 000000000..7bcec0873 --- /dev/null +++ b/python/eggroll/backport/proto/deepspeed_download_pb2.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: deepspeed_download.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 +import meta_pb2 as meta__pb2 +import containers_pb2 as containers__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x64\x65\x65pspeed_download.proto\x12\x1c\x63om.webank.eggroll.core.meta\x1a\x1egoogle/protobuf/duration.proto\x1a\nmeta.proto\x1a\x10\x63ontainers.proto\"\xad\x01\n\x16PrepareDownloadRequest\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\r\n\x05ranks\x18\x02 \x03(\x05\x12\x17\n\x0f\x63ompress_method\x18\x03 \x01(\t\x12\x16\n\x0e\x63ompress_level\x18\x04 \x01(\x05\x12?\n\x0c\x63ontent_type\x18\x05 \x01(\x0e\x32).com.webank.eggroll.core.meta.ContentType\">\n\x17PrepareDownloadResponse\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x02 \x01(\t\"\xa8\x01\n\x11\x44sDownloadRequest\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\r\n\x05ranks\x18\x02 \x03(\x05\x12\x17\n\x0f\x63ompress_method\x18\x03 \x01(\t\x12\x16\n\x0e\x63ompress_level\x18\x04 \x01(\x05\x12?\n\x0c\x63ontent_type\x18\x05 \x01(\x0e\x32).com.webank.eggroll.core.meta.ContentType\"s\n\x12\x44sDownloadResponse\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12I\n\x11\x63ontainer_content\x18\x02 \x03(\x0b\x32..com.webank.eggroll.core.meta.ContainerContent\"5\n\x17\x44sDownloadSplitResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\x0c\n\x04rank\x18\x02 \x01(\x05\x32\x85\x02\n\x11\x44sDownloadService\x12o\n\x08\x64ownload\x12/.com.webank.eggroll.core.meta.DsDownloadRequest\x1a\x30.com.webank.eggroll.core.meta.DsDownloadResponse\"\x00\x12\x7f\n\x11\x64ownload_by_split\x12/.com.webank.eggroll.core.meta.DsDownloadRequest\x1a\x35.com.webank.eggroll.core.meta.DsDownloadSplitResponse\"\x00\x30\x01\x62\x06proto3') + + + +_PREPAREDOWNLOADREQUEST = DESCRIPTOR.message_types_by_name['PrepareDownloadRequest'] +_PREPAREDOWNLOADRESPONSE = DESCRIPTOR.message_types_by_name['PrepareDownloadResponse'] +_DSDOWNLOADREQUEST = DESCRIPTOR.message_types_by_name['DsDownloadRequest'] +_DSDOWNLOADRESPONSE = DESCRIPTOR.message_types_by_name['DsDownloadResponse'] +_DSDOWNLOADSPLITRESPONSE = DESCRIPTOR.message_types_by_name['DsDownloadSplitResponse'] +PrepareDownloadRequest = _reflection.GeneratedProtocolMessageType('PrepareDownloadRequest', (_message.Message,), { + 'DESCRIPTOR' : _PREPAREDOWNLOADREQUEST, + '__module__' : 'deepspeed_download_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.PrepareDownloadRequest) + }) +_sym_db.RegisterMessage(PrepareDownloadRequest) + +PrepareDownloadResponse = _reflection.GeneratedProtocolMessageType('PrepareDownloadResponse', (_message.Message,), { + 'DESCRIPTOR' : _PREPAREDOWNLOADRESPONSE, + '__module__' : 'deepspeed_download_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.PrepareDownloadResponse) + }) +_sym_db.RegisterMessage(PrepareDownloadResponse) + +DsDownloadRequest = _reflection.GeneratedProtocolMessageType('DsDownloadRequest', (_message.Message,), { + 'DESCRIPTOR' : _DSDOWNLOADREQUEST, + '__module__' : 'deepspeed_download_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.DsDownloadRequest) + }) +_sym_db.RegisterMessage(DsDownloadRequest) + +DsDownloadResponse = _reflection.GeneratedProtocolMessageType('DsDownloadResponse', (_message.Message,), { + 'DESCRIPTOR' : _DSDOWNLOADRESPONSE, + '__module__' : 'deepspeed_download_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.DsDownloadResponse) + }) +_sym_db.RegisterMessage(DsDownloadResponse) + +DsDownloadSplitResponse = _reflection.GeneratedProtocolMessageType('DsDownloadSplitResponse', (_message.Message,), { + 'DESCRIPTOR' : _DSDOWNLOADSPLITRESPONSE, + '__module__' : 'deepspeed_download_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.DsDownloadSplitResponse) + }) +_sym_db.RegisterMessage(DsDownloadSplitResponse) + +_DSDOWNLOADSERVICE = DESCRIPTOR.services_by_name['DsDownloadService'] +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _PREPAREDOWNLOADREQUEST._serialized_start=121 + _PREPAREDOWNLOADREQUEST._serialized_end=294 + _PREPAREDOWNLOADRESPONSE._serialized_start=296 + _PREPAREDOWNLOADRESPONSE._serialized_end=358 + _DSDOWNLOADREQUEST._serialized_start=361 + _DSDOWNLOADREQUEST._serialized_end=529 + _DSDOWNLOADRESPONSE._serialized_start=531 + _DSDOWNLOADRESPONSE._serialized_end=646 + _DSDOWNLOADSPLITRESPONSE._serialized_start=648 + _DSDOWNLOADSPLITRESPONSE._serialized_end=701 + _DSDOWNLOADSERVICE._serialized_start=704 + _DSDOWNLOADSERVICE._serialized_end=965 +# @@protoc_insertion_point(module_scope) diff --git a/python/eggroll/backport/proto/deepspeed_download_pb2.pyi b/python/eggroll/backport/proto/deepspeed_download_pb2.pyi new file mode 100644 index 000000000..4b4fdfd67 --- /dev/null +++ b/python/eggroll/backport/proto/deepspeed_download_pb2.pyi @@ -0,0 +1,129 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" +import builtins +import collections.abc +import containers_pb2 +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.message +import sys + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing_extensions.final +class PrepareDownloadRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + RANKS_FIELD_NUMBER: builtins.int + COMPRESS_METHOD_FIELD_NUMBER: builtins.int + COMPRESS_LEVEL_FIELD_NUMBER: builtins.int + CONTENT_TYPE_FIELD_NUMBER: builtins.int + session_id: builtins.str + @property + def ranks(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ... + compress_method: builtins.str + compress_level: builtins.int + content_type: containers_pb2.ContentType.ValueType + def __init__( + self, + *, + session_id: builtins.str = ..., + ranks: collections.abc.Iterable[builtins.int] | None = ..., + compress_method: builtins.str = ..., + compress_level: builtins.int = ..., + content_type: containers_pb2.ContentType.ValueType = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["compress_level", b"compress_level", "compress_method", b"compress_method", "content_type", b"content_type", "ranks", b"ranks", "session_id", b"session_id"]) -> None: ... + +global___PrepareDownloadRequest = PrepareDownloadRequest + +@typing_extensions.final +class PrepareDownloadResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + CONTENT_FIELD_NUMBER: builtins.int + session_id: builtins.str + content: builtins.str + def __init__( + self, + *, + session_id: builtins.str = ..., + content: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["content", b"content", "session_id", b"session_id"]) -> None: ... + +global___PrepareDownloadResponse = PrepareDownloadResponse + +@typing_extensions.final +class DsDownloadRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + RANKS_FIELD_NUMBER: builtins.int + COMPRESS_METHOD_FIELD_NUMBER: builtins.int + COMPRESS_LEVEL_FIELD_NUMBER: builtins.int + CONTENT_TYPE_FIELD_NUMBER: builtins.int + session_id: builtins.str + @property + def ranks(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ... + compress_method: builtins.str + compress_level: builtins.int + content_type: containers_pb2.ContentType.ValueType + def __init__( + self, + *, + session_id: builtins.str = ..., + ranks: collections.abc.Iterable[builtins.int] | None = ..., + compress_method: builtins.str = ..., + compress_level: builtins.int = ..., + content_type: containers_pb2.ContentType.ValueType = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["compress_level", b"compress_level", "compress_method", b"compress_method", "content_type", b"content_type", "ranks", b"ranks", "session_id", b"session_id"]) -> None: ... + +global___DsDownloadRequest = DsDownloadRequest + +@typing_extensions.final +class DsDownloadResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + CONTAINER_CONTENT_FIELD_NUMBER: builtins.int + session_id: builtins.str + @property + def container_content(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[containers_pb2.ContainerContent]: ... + def __init__( + self, + *, + session_id: builtins.str = ..., + container_content: collections.abc.Iterable[containers_pb2.ContainerContent] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["container_content", b"container_content", "session_id", b"session_id"]) -> None: ... + +global___DsDownloadResponse = DsDownloadResponse + +@typing_extensions.final +class DsDownloadSplitResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DATA_FIELD_NUMBER: builtins.int + RANK_FIELD_NUMBER: builtins.int + data: builtins.bytes + rank: builtins.int + def __init__( + self, + *, + data: builtins.bytes = ..., + rank: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["data", b"data", "rank", b"rank"]) -> None: ... + +global___DsDownloadSplitResponse = DsDownloadSplitResponse diff --git a/python/eggroll/backport/proto/deepspeed_download_pb2_grpc.py b/python/eggroll/backport/proto/deepspeed_download_pb2_grpc.py new file mode 100644 index 000000000..4edf98673 --- /dev/null +++ b/python/eggroll/backport/proto/deepspeed_download_pb2_grpc.py @@ -0,0 +1,99 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +import deepspeed_download_pb2 as deepspeed__download__pb2 + + +class DsDownloadServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.download = channel.unary_unary( + '/com.webank.eggroll.core.meta.DsDownloadService/download', + request_serializer=deepspeed__download__pb2.DsDownloadRequest.SerializeToString, + response_deserializer=deepspeed__download__pb2.DsDownloadResponse.FromString, + ) + self.download_by_split = channel.unary_stream( + '/com.webank.eggroll.core.meta.DsDownloadService/download_by_split', + request_serializer=deepspeed__download__pb2.DsDownloadRequest.SerializeToString, + response_deserializer=deepspeed__download__pb2.DsDownloadSplitResponse.FromString, + ) + + +class DsDownloadServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def download(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def download_by_split(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_DsDownloadServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'download': grpc.unary_unary_rpc_method_handler( + servicer.download, + request_deserializer=deepspeed__download__pb2.DsDownloadRequest.FromString, + response_serializer=deepspeed__download__pb2.DsDownloadResponse.SerializeToString, + ), + 'download_by_split': grpc.unary_stream_rpc_method_handler( + servicer.download_by_split, + request_deserializer=deepspeed__download__pb2.DsDownloadRequest.FromString, + response_serializer=deepspeed__download__pb2.DsDownloadSplitResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'com.webank.eggroll.core.meta.DsDownloadService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class DsDownloadService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def download(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/com.webank.eggroll.core.meta.DsDownloadService/download', + deepspeed__download__pb2.DsDownloadRequest.SerializeToString, + deepspeed__download__pb2.DsDownloadResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def download_by_split(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/com.webank.eggroll.core.meta.DsDownloadService/download_by_split', + deepspeed__download__pb2.DsDownloadRequest.SerializeToString, + deepspeed__download__pb2.DsDownloadSplitResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/python/eggroll/backport/proto/deepspeed_pb2.py b/python/eggroll/backport/proto/deepspeed_pb2.py new file mode 100644 index 000000000..b391f9a3e --- /dev/null +++ b/python/eggroll/backport/proto/deepspeed_pb2.py @@ -0,0 +1,372 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: deepspeed.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 +import meta_pb2 as meta__pb2 +import containers_pb2 as containers__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64\x65\x65pspeed.proto\x12\x1c\x63om.webank.eggroll.core.meta\x1a\x1egoogle/protobuf/duration.proto\x1a\nmeta.proto\x1a\x10\x63ontainers.proto\"=\n\x0fStoreSetRequest\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\r\n\x05value\x18\x03 \x01(\x0c\"\x12\n\x10StoreSetResponse\"Z\n\x0fStoreGetRequest\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12*\n\x07timeout\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\"5\n\x10StoreGetResponse\x12\r\n\x05value\x18\x01 \x01(\x0c\x12\x12\n\nis_timeout\x18\x02 \x01(\x08\">\n\x0fStoreAddRequest\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x0e\n\x06\x61mount\x18\x03 \x01(\x03\"\"\n\x10StoreAddResponse\x12\x0e\n\x06\x61mount\x18\x01 \x01(\x03\"X\n\x19StoreCompareAndSetRequest\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x0e\n\x06\x65xpect\x18\x03 \x01(\t\x12\x0e\n\x06update\x18\x04 \x01(\t\"\x1c\n\x1aStoreCompareAndSetResponse\"[\n\x10StoreWaitRequest\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\t\x12*\n\x07timeout\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\"\x13\n\x11StoreWaitResponse\"%\n\x13StoreNumKeysRequest\x12\x0e\n\x06prefix\x18\x01 \x01(\t\"(\n\x14StoreNumKeysResponse\x12\x10\n\x08num_keys\x18\x01 \x01(\x03\"4\n\x15StoreDeleteKeyRequest\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\t\")\n\x16StoreDeleteKeyResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\"%\n\x13StoreDestroyRequest\x12\x0e\n\x06prefix\x18\x01 \x01(\t\"\'\n\x14StoreDestroyResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\"\xe5\x05\n\x10SubmitJobRequest\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x10\n\x08job_type\x18\x03 \x01(\t\x12\x12\n\nworld_size\x18\x04 \x01(\r\x12\x19\n\x11\x63ommand_arguments\x18\x05 \x03(\t\x12g\n\x15\x65nvironment_variables\x18\x06 \x03(\x0b\x32H.com.webank.eggroll.core.meta.SubmitJobRequest.EnvironmentVariablesEntry\x12H\n\x05\x66iles\x18\x07 \x03(\x0b\x32\x39.com.webank.eggroll.core.meta.SubmitJobRequest.FilesEntry\x12U\n\x0czipped_files\x18\x08 \x03(\x0b\x32?.com.webank.eggroll.core.meta.SubmitJobRequest.ZippedFilesEntry\x12G\n\x10resource_options\x18\t \x01(\x0b\x32-.com.webank.eggroll.core.meta.ResourceOptions\x12L\n\x07options\x18\x13 \x03(\x0b\x32;.com.webank.eggroll.core.meta.SubmitJobRequest.OptionsEntry\x1a;\n\x19\x45nvironmentVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a,\n\nFilesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a\x32\n\x10ZippedFilesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c:\x02\x38\x01\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"O\n\x0fResourceOptions\x12\x17\n\x0ftimeout_seconds\x18\x01 \x01(\r\x12#\n\x1bresource_exhausted_strategy\x18\x02 \x01(\t\"d\n\x11SubmitJobResponse\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12;\n\nprocessors\x18\x02 \x03(\x0b\x32\'.com.webank.eggroll.core.meta.Processor\"$\n\x0eKillJobRequest\x12\x12\n\nsession_id\x18\x01 \x01(\t\"%\n\x0fKillJobResponse\x12\x12\n\nsession_id\x18\x01 \x01(\t\"$\n\x0eStopJobRequest\x12\x12\n\nsession_id\x18\x01 \x01(\t\"%\n\x0fStopJobResponse\x12\x12\n\nsession_id\x18\x01 \x01(\t\"%\n\x0fQueryJobRequest\x12\x12\n\nsession_id\x18\x01 \x01(\t\"\x85\x01\n\x10QueryJobResponse\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\x10\n\x08job_type\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12;\n\nprocessors\x18\x05 \x03(\x0b\x32\'.com.webank.eggroll.core.meta.Processor\"+\n\x15QueryJobStatusRequest\x12\x12\n\nsession_id\x18\x01 \x01(\t\"<\n\x16QueryJobStatusResponse\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\t\"\xa9\x01\n\x12\x44ownloadJobRequest\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12\r\n\x05ranks\x18\x02 \x03(\x05\x12\x17\n\x0f\x63ompress_method\x18\x03 \x01(\t\x12\x16\n\x0e\x63ompress_level\x18\x04 \x01(\x05\x12?\n\x0c\x63ontent_type\x18\x05 \x01(\x0e\x32).com.webank.eggroll.core.meta.ContentType\"t\n\x13\x44ownloadJobResponse\x12\x12\n\nsession_id\x18\x01 \x01(\t\x12I\n\x11\x63ontainer_content\x18\x02 \x03(\x0b\x32..com.webank.eggroll.core.meta.ContainerContent2\xb2\x06\n\x18\x44\x65\x65pspeedRendezvousStore\x12\x66\n\x03Set\x12-.com.webank.eggroll.core.meta.StoreSetRequest\x1a..com.webank.eggroll.core.meta.StoreSetResponse\"\x00\x12\x66\n\x03Get\x12-.com.webank.eggroll.core.meta.StoreGetRequest\x1a..com.webank.eggroll.core.meta.StoreGetResponse\"\x00\x12\x66\n\x03\x41\x64\x64\x12-.com.webank.eggroll.core.meta.StoreAddRequest\x1a..com.webank.eggroll.core.meta.StoreAddResponse\"\x00\x12\x84\x01\n\rCompareAndSet\x12\x37.com.webank.eggroll.core.meta.StoreCompareAndSetRequest\x1a\x38.com.webank.eggroll.core.meta.StoreCompareAndSetResponse\"\x00\x12i\n\x04Wait\x12..com.webank.eggroll.core.meta.StoreWaitRequest\x1a/.com.webank.eggroll.core.meta.StoreWaitResponse\"\x00\x12r\n\x07NumKeys\x12\x31.com.webank.eggroll.core.meta.StoreNumKeysRequest\x1a\x32.com.webank.eggroll.core.meta.StoreNumKeysResponse\"\x00\x12x\n\tDeleteKey\x12\x33.com.webank.eggroll.core.meta.StoreDeleteKeyRequest\x1a\x34.com.webank.eggroll.core.meta.StoreDeleteKeyResponse\"\x00\x62\x06proto3') + + + +_STORESETREQUEST = DESCRIPTOR.message_types_by_name['StoreSetRequest'] +_STORESETRESPONSE = DESCRIPTOR.message_types_by_name['StoreSetResponse'] +_STOREGETREQUEST = DESCRIPTOR.message_types_by_name['StoreGetRequest'] +_STOREGETRESPONSE = DESCRIPTOR.message_types_by_name['StoreGetResponse'] +_STOREADDREQUEST = DESCRIPTOR.message_types_by_name['StoreAddRequest'] +_STOREADDRESPONSE = DESCRIPTOR.message_types_by_name['StoreAddResponse'] +_STORECOMPAREANDSETREQUEST = DESCRIPTOR.message_types_by_name['StoreCompareAndSetRequest'] +_STORECOMPAREANDSETRESPONSE = DESCRIPTOR.message_types_by_name['StoreCompareAndSetResponse'] +_STOREWAITREQUEST = DESCRIPTOR.message_types_by_name['StoreWaitRequest'] +_STOREWAITRESPONSE = DESCRIPTOR.message_types_by_name['StoreWaitResponse'] +_STORENUMKEYSREQUEST = DESCRIPTOR.message_types_by_name['StoreNumKeysRequest'] +_STORENUMKEYSRESPONSE = DESCRIPTOR.message_types_by_name['StoreNumKeysResponse'] +_STOREDELETEKEYREQUEST = DESCRIPTOR.message_types_by_name['StoreDeleteKeyRequest'] +_STOREDELETEKEYRESPONSE = DESCRIPTOR.message_types_by_name['StoreDeleteKeyResponse'] +_STOREDESTROYREQUEST = DESCRIPTOR.message_types_by_name['StoreDestroyRequest'] +_STOREDESTROYRESPONSE = DESCRIPTOR.message_types_by_name['StoreDestroyResponse'] +_SUBMITJOBREQUEST = DESCRIPTOR.message_types_by_name['SubmitJobRequest'] +_SUBMITJOBREQUEST_ENVIRONMENTVARIABLESENTRY = _SUBMITJOBREQUEST.nested_types_by_name['EnvironmentVariablesEntry'] +_SUBMITJOBREQUEST_FILESENTRY = _SUBMITJOBREQUEST.nested_types_by_name['FilesEntry'] +_SUBMITJOBREQUEST_ZIPPEDFILESENTRY = _SUBMITJOBREQUEST.nested_types_by_name['ZippedFilesEntry'] +_SUBMITJOBREQUEST_OPTIONSENTRY = _SUBMITJOBREQUEST.nested_types_by_name['OptionsEntry'] +_RESOURCEOPTIONS = DESCRIPTOR.message_types_by_name['ResourceOptions'] +_SUBMITJOBRESPONSE = DESCRIPTOR.message_types_by_name['SubmitJobResponse'] +_KILLJOBREQUEST = DESCRIPTOR.message_types_by_name['KillJobRequest'] +_KILLJOBRESPONSE = DESCRIPTOR.message_types_by_name['KillJobResponse'] +_STOPJOBREQUEST = DESCRIPTOR.message_types_by_name['StopJobRequest'] +_STOPJOBRESPONSE = DESCRIPTOR.message_types_by_name['StopJobResponse'] +_QUERYJOBREQUEST = DESCRIPTOR.message_types_by_name['QueryJobRequest'] +_QUERYJOBRESPONSE = DESCRIPTOR.message_types_by_name['QueryJobResponse'] +_QUERYJOBSTATUSREQUEST = DESCRIPTOR.message_types_by_name['QueryJobStatusRequest'] +_QUERYJOBSTATUSRESPONSE = DESCRIPTOR.message_types_by_name['QueryJobStatusResponse'] +_DOWNLOADJOBREQUEST = DESCRIPTOR.message_types_by_name['DownloadJobRequest'] +_DOWNLOADJOBRESPONSE = DESCRIPTOR.message_types_by_name['DownloadJobResponse'] +StoreSetRequest = _reflection.GeneratedProtocolMessageType('StoreSetRequest', (_message.Message,), { + 'DESCRIPTOR' : _STORESETREQUEST, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreSetRequest) + }) +_sym_db.RegisterMessage(StoreSetRequest) + +StoreSetResponse = _reflection.GeneratedProtocolMessageType('StoreSetResponse', (_message.Message,), { + 'DESCRIPTOR' : _STORESETRESPONSE, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreSetResponse) + }) +_sym_db.RegisterMessage(StoreSetResponse) + +StoreGetRequest = _reflection.GeneratedProtocolMessageType('StoreGetRequest', (_message.Message,), { + 'DESCRIPTOR' : _STOREGETREQUEST, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreGetRequest) + }) +_sym_db.RegisterMessage(StoreGetRequest) + +StoreGetResponse = _reflection.GeneratedProtocolMessageType('StoreGetResponse', (_message.Message,), { + 'DESCRIPTOR' : _STOREGETRESPONSE, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreGetResponse) + }) +_sym_db.RegisterMessage(StoreGetResponse) + +StoreAddRequest = _reflection.GeneratedProtocolMessageType('StoreAddRequest', (_message.Message,), { + 'DESCRIPTOR' : _STOREADDREQUEST, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreAddRequest) + }) +_sym_db.RegisterMessage(StoreAddRequest) + +StoreAddResponse = _reflection.GeneratedProtocolMessageType('StoreAddResponse', (_message.Message,), { + 'DESCRIPTOR' : _STOREADDRESPONSE, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreAddResponse) + }) +_sym_db.RegisterMessage(StoreAddResponse) + +StoreCompareAndSetRequest = _reflection.GeneratedProtocolMessageType('StoreCompareAndSetRequest', (_message.Message,), { + 'DESCRIPTOR' : _STORECOMPAREANDSETREQUEST, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreCompareAndSetRequest) + }) +_sym_db.RegisterMessage(StoreCompareAndSetRequest) + +StoreCompareAndSetResponse = _reflection.GeneratedProtocolMessageType('StoreCompareAndSetResponse', (_message.Message,), { + 'DESCRIPTOR' : _STORECOMPAREANDSETRESPONSE, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreCompareAndSetResponse) + }) +_sym_db.RegisterMessage(StoreCompareAndSetResponse) + +StoreWaitRequest = _reflection.GeneratedProtocolMessageType('StoreWaitRequest', (_message.Message,), { + 'DESCRIPTOR' : _STOREWAITREQUEST, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreWaitRequest) + }) +_sym_db.RegisterMessage(StoreWaitRequest) + +StoreWaitResponse = _reflection.GeneratedProtocolMessageType('StoreWaitResponse', (_message.Message,), { + 'DESCRIPTOR' : _STOREWAITRESPONSE, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreWaitResponse) + }) +_sym_db.RegisterMessage(StoreWaitResponse) + +StoreNumKeysRequest = _reflection.GeneratedProtocolMessageType('StoreNumKeysRequest', (_message.Message,), { + 'DESCRIPTOR' : _STORENUMKEYSREQUEST, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreNumKeysRequest) + }) +_sym_db.RegisterMessage(StoreNumKeysRequest) + +StoreNumKeysResponse = _reflection.GeneratedProtocolMessageType('StoreNumKeysResponse', (_message.Message,), { + 'DESCRIPTOR' : _STORENUMKEYSRESPONSE, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreNumKeysResponse) + }) +_sym_db.RegisterMessage(StoreNumKeysResponse) + +StoreDeleteKeyRequest = _reflection.GeneratedProtocolMessageType('StoreDeleteKeyRequest', (_message.Message,), { + 'DESCRIPTOR' : _STOREDELETEKEYREQUEST, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreDeleteKeyRequest) + }) +_sym_db.RegisterMessage(StoreDeleteKeyRequest) + +StoreDeleteKeyResponse = _reflection.GeneratedProtocolMessageType('StoreDeleteKeyResponse', (_message.Message,), { + 'DESCRIPTOR' : _STOREDELETEKEYRESPONSE, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreDeleteKeyResponse) + }) +_sym_db.RegisterMessage(StoreDeleteKeyResponse) + +StoreDestroyRequest = _reflection.GeneratedProtocolMessageType('StoreDestroyRequest', (_message.Message,), { + 'DESCRIPTOR' : _STOREDESTROYREQUEST, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreDestroyRequest) + }) +_sym_db.RegisterMessage(StoreDestroyRequest) + +StoreDestroyResponse = _reflection.GeneratedProtocolMessageType('StoreDestroyResponse', (_message.Message,), { + 'DESCRIPTOR' : _STOREDESTROYRESPONSE, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreDestroyResponse) + }) +_sym_db.RegisterMessage(StoreDestroyResponse) + +SubmitJobRequest = _reflection.GeneratedProtocolMessageType('SubmitJobRequest', (_message.Message,), { + + 'EnvironmentVariablesEntry' : _reflection.GeneratedProtocolMessageType('EnvironmentVariablesEntry', (_message.Message,), { + 'DESCRIPTOR' : _SUBMITJOBREQUEST_ENVIRONMENTVARIABLESENTRY, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.SubmitJobRequest.EnvironmentVariablesEntry) + }) + , + + 'FilesEntry' : _reflection.GeneratedProtocolMessageType('FilesEntry', (_message.Message,), { + 'DESCRIPTOR' : _SUBMITJOBREQUEST_FILESENTRY, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.SubmitJobRequest.FilesEntry) + }) + , + + 'ZippedFilesEntry' : _reflection.GeneratedProtocolMessageType('ZippedFilesEntry', (_message.Message,), { + 'DESCRIPTOR' : _SUBMITJOBREQUEST_ZIPPEDFILESENTRY, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.SubmitJobRequest.ZippedFilesEntry) + }) + , + + 'OptionsEntry' : _reflection.GeneratedProtocolMessageType('OptionsEntry', (_message.Message,), { + 'DESCRIPTOR' : _SUBMITJOBREQUEST_OPTIONSENTRY, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.SubmitJobRequest.OptionsEntry) + }) + , + 'DESCRIPTOR' : _SUBMITJOBREQUEST, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.SubmitJobRequest) + }) +_sym_db.RegisterMessage(SubmitJobRequest) +_sym_db.RegisterMessage(SubmitJobRequest.EnvironmentVariablesEntry) +_sym_db.RegisterMessage(SubmitJobRequest.FilesEntry) +_sym_db.RegisterMessage(SubmitJobRequest.ZippedFilesEntry) +_sym_db.RegisterMessage(SubmitJobRequest.OptionsEntry) + +ResourceOptions = _reflection.GeneratedProtocolMessageType('ResourceOptions', (_message.Message,), { + 'DESCRIPTOR' : _RESOURCEOPTIONS, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.ResourceOptions) + }) +_sym_db.RegisterMessage(ResourceOptions) + +SubmitJobResponse = _reflection.GeneratedProtocolMessageType('SubmitJobResponse', (_message.Message,), { + 'DESCRIPTOR' : _SUBMITJOBRESPONSE, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.SubmitJobResponse) + }) +_sym_db.RegisterMessage(SubmitJobResponse) + +KillJobRequest = _reflection.GeneratedProtocolMessageType('KillJobRequest', (_message.Message,), { + 'DESCRIPTOR' : _KILLJOBREQUEST, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.KillJobRequest) + }) +_sym_db.RegisterMessage(KillJobRequest) + +KillJobResponse = _reflection.GeneratedProtocolMessageType('KillJobResponse', (_message.Message,), { + 'DESCRIPTOR' : _KILLJOBRESPONSE, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.KillJobResponse) + }) +_sym_db.RegisterMessage(KillJobResponse) + +StopJobRequest = _reflection.GeneratedProtocolMessageType('StopJobRequest', (_message.Message,), { + 'DESCRIPTOR' : _STOPJOBREQUEST, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StopJobRequest) + }) +_sym_db.RegisterMessage(StopJobRequest) + +StopJobResponse = _reflection.GeneratedProtocolMessageType('StopJobResponse', (_message.Message,), { + 'DESCRIPTOR' : _STOPJOBRESPONSE, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StopJobResponse) + }) +_sym_db.RegisterMessage(StopJobResponse) + +QueryJobRequest = _reflection.GeneratedProtocolMessageType('QueryJobRequest', (_message.Message,), { + 'DESCRIPTOR' : _QUERYJOBREQUEST, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.QueryJobRequest) + }) +_sym_db.RegisterMessage(QueryJobRequest) + +QueryJobResponse = _reflection.GeneratedProtocolMessageType('QueryJobResponse', (_message.Message,), { + 'DESCRIPTOR' : _QUERYJOBRESPONSE, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.QueryJobResponse) + }) +_sym_db.RegisterMessage(QueryJobResponse) + +QueryJobStatusRequest = _reflection.GeneratedProtocolMessageType('QueryJobStatusRequest', (_message.Message,), { + 'DESCRIPTOR' : _QUERYJOBSTATUSREQUEST, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.QueryJobStatusRequest) + }) +_sym_db.RegisterMessage(QueryJobStatusRequest) + +QueryJobStatusResponse = _reflection.GeneratedProtocolMessageType('QueryJobStatusResponse', (_message.Message,), { + 'DESCRIPTOR' : _QUERYJOBSTATUSRESPONSE, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.QueryJobStatusResponse) + }) +_sym_db.RegisterMessage(QueryJobStatusResponse) + +DownloadJobRequest = _reflection.GeneratedProtocolMessageType('DownloadJobRequest', (_message.Message,), { + 'DESCRIPTOR' : _DOWNLOADJOBREQUEST, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.DownloadJobRequest) + }) +_sym_db.RegisterMessage(DownloadJobRequest) + +DownloadJobResponse = _reflection.GeneratedProtocolMessageType('DownloadJobResponse', (_message.Message,), { + 'DESCRIPTOR' : _DOWNLOADJOBRESPONSE, + '__module__' : 'deepspeed_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.DownloadJobResponse) + }) +_sym_db.RegisterMessage(DownloadJobResponse) + +_DEEPSPEEDRENDEZVOUSSTORE = DESCRIPTOR.services_by_name['DeepspeedRendezvousStore'] +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _SUBMITJOBREQUEST_ENVIRONMENTVARIABLESENTRY._options = None + _SUBMITJOBREQUEST_ENVIRONMENTVARIABLESENTRY._serialized_options = b'8\001' + _SUBMITJOBREQUEST_FILESENTRY._options = None + _SUBMITJOBREQUEST_FILESENTRY._serialized_options = b'8\001' + _SUBMITJOBREQUEST_ZIPPEDFILESENTRY._options = None + _SUBMITJOBREQUEST_ZIPPEDFILESENTRY._serialized_options = b'8\001' + _SUBMITJOBREQUEST_OPTIONSENTRY._options = None + _SUBMITJOBREQUEST_OPTIONSENTRY._serialized_options = b'8\001' + _STORESETREQUEST._serialized_start=111 + _STORESETREQUEST._serialized_end=172 + _STORESETRESPONSE._serialized_start=174 + _STORESETRESPONSE._serialized_end=192 + _STOREGETREQUEST._serialized_start=194 + _STOREGETREQUEST._serialized_end=284 + _STOREGETRESPONSE._serialized_start=286 + _STOREGETRESPONSE._serialized_end=339 + _STOREADDREQUEST._serialized_start=341 + _STOREADDREQUEST._serialized_end=403 + _STOREADDRESPONSE._serialized_start=405 + _STOREADDRESPONSE._serialized_end=439 + _STORECOMPAREANDSETREQUEST._serialized_start=441 + _STORECOMPAREANDSETREQUEST._serialized_end=529 + _STORECOMPAREANDSETRESPONSE._serialized_start=531 + _STORECOMPAREANDSETRESPONSE._serialized_end=559 + _STOREWAITREQUEST._serialized_start=561 + _STOREWAITREQUEST._serialized_end=652 + _STOREWAITRESPONSE._serialized_start=654 + _STOREWAITRESPONSE._serialized_end=673 + _STORENUMKEYSREQUEST._serialized_start=675 + _STORENUMKEYSREQUEST._serialized_end=712 + _STORENUMKEYSRESPONSE._serialized_start=714 + _STORENUMKEYSRESPONSE._serialized_end=754 + _STOREDELETEKEYREQUEST._serialized_start=756 + _STOREDELETEKEYREQUEST._serialized_end=808 + _STOREDELETEKEYRESPONSE._serialized_start=810 + _STOREDELETEKEYRESPONSE._serialized_end=851 + _STOREDESTROYREQUEST._serialized_start=853 + _STOREDESTROYREQUEST._serialized_end=890 + _STOREDESTROYRESPONSE._serialized_start=892 + _STOREDESTROYRESPONSE._serialized_end=931 + _SUBMITJOBREQUEST._serialized_start=934 + _SUBMITJOBREQUEST._serialized_end=1675 + _SUBMITJOBREQUEST_ENVIRONMENTVARIABLESENTRY._serialized_start=1470 + _SUBMITJOBREQUEST_ENVIRONMENTVARIABLESENTRY._serialized_end=1529 + _SUBMITJOBREQUEST_FILESENTRY._serialized_start=1531 + _SUBMITJOBREQUEST_FILESENTRY._serialized_end=1575 + _SUBMITJOBREQUEST_ZIPPEDFILESENTRY._serialized_start=1577 + _SUBMITJOBREQUEST_ZIPPEDFILESENTRY._serialized_end=1627 + _SUBMITJOBREQUEST_OPTIONSENTRY._serialized_start=1629 + _SUBMITJOBREQUEST_OPTIONSENTRY._serialized_end=1675 + _RESOURCEOPTIONS._serialized_start=1677 + _RESOURCEOPTIONS._serialized_end=1756 + _SUBMITJOBRESPONSE._serialized_start=1758 + _SUBMITJOBRESPONSE._serialized_end=1858 + _KILLJOBREQUEST._serialized_start=1860 + _KILLJOBREQUEST._serialized_end=1896 + _KILLJOBRESPONSE._serialized_start=1898 + _KILLJOBRESPONSE._serialized_end=1935 + _STOPJOBREQUEST._serialized_start=1937 + _STOPJOBREQUEST._serialized_end=1973 + _STOPJOBRESPONSE._serialized_start=1975 + _STOPJOBRESPONSE._serialized_end=2012 + _QUERYJOBREQUEST._serialized_start=2014 + _QUERYJOBREQUEST._serialized_end=2051 + _QUERYJOBRESPONSE._serialized_start=2054 + _QUERYJOBRESPONSE._serialized_end=2187 + _QUERYJOBSTATUSREQUEST._serialized_start=2189 + _QUERYJOBSTATUSREQUEST._serialized_end=2232 + _QUERYJOBSTATUSRESPONSE._serialized_start=2234 + _QUERYJOBSTATUSRESPONSE._serialized_end=2294 + _DOWNLOADJOBREQUEST._serialized_start=2297 + _DOWNLOADJOBREQUEST._serialized_end=2466 + _DOWNLOADJOBRESPONSE._serialized_start=2468 + _DOWNLOADJOBRESPONSE._serialized_end=2584 + _DEEPSPEEDRENDEZVOUSSTORE._serialized_start=2587 + _DEEPSPEEDRENDEZVOUSSTORE._serialized_end=3405 +# @@protoc_insertion_point(module_scope) diff --git a/python/eggroll/backport/proto/deepspeed_pb2.pyi b/python/eggroll/backport/proto/deepspeed_pb2.pyi new file mode 100644 index 000000000..d49e38aa3 --- /dev/null +++ b/python/eggroll/backport/proto/deepspeed_pb2.pyi @@ -0,0 +1,621 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" +import builtins +import collections.abc +import containers_pb2 +import google.protobuf.descriptor +import google.protobuf.duration_pb2 +import google.protobuf.internal.containers +import google.protobuf.message +import meta_pb2 +import sys + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing_extensions.final +class StoreSetRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + PREFIX_FIELD_NUMBER: builtins.int + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + prefix: builtins.str + key: builtins.bytes + value: builtins.bytes + def __init__( + self, + *, + prefix: builtins.str = ..., + key: builtins.bytes = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "prefix", b"prefix", "value", b"value"]) -> None: ... + +global___StoreSetRequest = StoreSetRequest + +@typing_extensions.final +class StoreSetResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___StoreSetResponse = StoreSetResponse + +@typing_extensions.final +class StoreGetRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + PREFIX_FIELD_NUMBER: builtins.int + KEY_FIELD_NUMBER: builtins.int + TIMEOUT_FIELD_NUMBER: builtins.int + prefix: builtins.str + key: builtins.bytes + @property + def timeout(self) -> google.protobuf.duration_pb2.Duration: ... + def __init__( + self, + *, + prefix: builtins.str = ..., + key: builtins.bytes = ..., + timeout: google.protobuf.duration_pb2.Duration | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["timeout", b"timeout"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "prefix", b"prefix", "timeout", b"timeout"]) -> None: ... + +global___StoreGetRequest = StoreGetRequest + +@typing_extensions.final +class StoreGetResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + VALUE_FIELD_NUMBER: builtins.int + IS_TIMEOUT_FIELD_NUMBER: builtins.int + value: builtins.bytes + is_timeout: builtins.bool + def __init__( + self, + *, + value: builtins.bytes = ..., + is_timeout: builtins.bool = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["is_timeout", b"is_timeout", "value", b"value"]) -> None: ... + +global___StoreGetResponse = StoreGetResponse + +@typing_extensions.final +class StoreAddRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + PREFIX_FIELD_NUMBER: builtins.int + KEY_FIELD_NUMBER: builtins.int + AMOUNT_FIELD_NUMBER: builtins.int + prefix: builtins.str + key: builtins.bytes + amount: builtins.int + def __init__( + self, + *, + prefix: builtins.str = ..., + key: builtins.bytes = ..., + amount: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["amount", b"amount", "key", b"key", "prefix", b"prefix"]) -> None: ... + +global___StoreAddRequest = StoreAddRequest + +@typing_extensions.final +class StoreAddResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + AMOUNT_FIELD_NUMBER: builtins.int + amount: builtins.int + def __init__( + self, + *, + amount: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["amount", b"amount"]) -> None: ... + +global___StoreAddResponse = StoreAddResponse + +@typing_extensions.final +class StoreCompareAndSetRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + PREFIX_FIELD_NUMBER: builtins.int + KEY_FIELD_NUMBER: builtins.int + EXPECT_FIELD_NUMBER: builtins.int + UPDATE_FIELD_NUMBER: builtins.int + prefix: builtins.str + key: builtins.str + expect: builtins.str + update: builtins.str + def __init__( + self, + *, + prefix: builtins.str = ..., + key: builtins.str = ..., + expect: builtins.str = ..., + update: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["expect", b"expect", "key", b"key", "prefix", b"prefix", "update", b"update"]) -> None: ... + +global___StoreCompareAndSetRequest = StoreCompareAndSetRequest + +@typing_extensions.final +class StoreCompareAndSetResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___StoreCompareAndSetResponse = StoreCompareAndSetResponse + +@typing_extensions.final +class StoreWaitRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + PREFIX_FIELD_NUMBER: builtins.int + KEY_FIELD_NUMBER: builtins.int + TIMEOUT_FIELD_NUMBER: builtins.int + prefix: builtins.str + key: builtins.str + @property + def timeout(self) -> google.protobuf.duration_pb2.Duration: ... + def __init__( + self, + *, + prefix: builtins.str = ..., + key: builtins.str = ..., + timeout: google.protobuf.duration_pb2.Duration | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["timeout", b"timeout"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "prefix", b"prefix", "timeout", b"timeout"]) -> None: ... + +global___StoreWaitRequest = StoreWaitRequest + +@typing_extensions.final +class StoreWaitResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___StoreWaitResponse = StoreWaitResponse + +@typing_extensions.final +class StoreNumKeysRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + PREFIX_FIELD_NUMBER: builtins.int + prefix: builtins.str + def __init__( + self, + *, + prefix: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["prefix", b"prefix"]) -> None: ... + +global___StoreNumKeysRequest = StoreNumKeysRequest + +@typing_extensions.final +class StoreNumKeysResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NUM_KEYS_FIELD_NUMBER: builtins.int + num_keys: builtins.int + def __init__( + self, + *, + num_keys: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["num_keys", b"num_keys"]) -> None: ... + +global___StoreNumKeysResponse = StoreNumKeysResponse + +@typing_extensions.final +class StoreDeleteKeyRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + PREFIX_FIELD_NUMBER: builtins.int + KEY_FIELD_NUMBER: builtins.int + prefix: builtins.str + key: builtins.str + def __init__( + self, + *, + prefix: builtins.str = ..., + key: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "prefix", b"prefix"]) -> None: ... + +global___StoreDeleteKeyRequest = StoreDeleteKeyRequest + +@typing_extensions.final +class StoreDeleteKeyResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SUCCESS_FIELD_NUMBER: builtins.int + success: builtins.bool + def __init__( + self, + *, + success: builtins.bool = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["success", b"success"]) -> None: ... + +global___StoreDeleteKeyResponse = StoreDeleteKeyResponse + +@typing_extensions.final +class StoreDestroyRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + PREFIX_FIELD_NUMBER: builtins.int + prefix: builtins.str + def __init__( + self, + *, + prefix: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["prefix", b"prefix"]) -> None: ... + +global___StoreDestroyRequest = StoreDestroyRequest + +@typing_extensions.final +class StoreDestroyResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SUCCESS_FIELD_NUMBER: builtins.int + success: builtins.bool + def __init__( + self, + *, + success: builtins.bool = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["success", b"success"]) -> None: ... + +global___StoreDestroyResponse = StoreDestroyResponse + +@typing_extensions.final +class SubmitJobRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class EnvironmentVariablesEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + @typing_extensions.final + class FilesEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.bytes + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + @typing_extensions.final + class ZippedFilesEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.bytes + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + @typing_extensions.final + class OptionsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + SESSION_ID_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + JOB_TYPE_FIELD_NUMBER: builtins.int + WORLD_SIZE_FIELD_NUMBER: builtins.int + COMMAND_ARGUMENTS_FIELD_NUMBER: builtins.int + ENVIRONMENT_VARIABLES_FIELD_NUMBER: builtins.int + FILES_FIELD_NUMBER: builtins.int + ZIPPED_FILES_FIELD_NUMBER: builtins.int + RESOURCE_OPTIONS_FIELD_NUMBER: builtins.int + OPTIONS_FIELD_NUMBER: builtins.int + session_id: builtins.str + name: builtins.str + job_type: builtins.str + world_size: builtins.int + @property + def command_arguments(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ... + @property + def environment_variables(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ... + @property + def files(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.bytes]: ... + @property + def zipped_files(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.bytes]: ... + @property + def resource_options(self) -> global___ResourceOptions: + """concrete options""" + @property + def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: + """extra options""" + def __init__( + self, + *, + session_id: builtins.str = ..., + name: builtins.str = ..., + job_type: builtins.str = ..., + world_size: builtins.int = ..., + command_arguments: collections.abc.Iterable[builtins.str] | None = ..., + environment_variables: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + files: collections.abc.Mapping[builtins.str, builtins.bytes] | None = ..., + zipped_files: collections.abc.Mapping[builtins.str, builtins.bytes] | None = ..., + resource_options: global___ResourceOptions | None = ..., + options: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["resource_options", b"resource_options"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["command_arguments", b"command_arguments", "environment_variables", b"environment_variables", "files", b"files", "job_type", b"job_type", "name", b"name", "options", b"options", "resource_options", b"resource_options", "session_id", b"session_id", "world_size", b"world_size", "zipped_files", b"zipped_files"]) -> None: ... + +global___SubmitJobRequest = SubmitJobRequest + +@typing_extensions.final +class ResourceOptions(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TIMEOUT_SECONDS_FIELD_NUMBER: builtins.int + RESOURCE_EXHAUSTED_STRATEGY_FIELD_NUMBER: builtins.int + timeout_seconds: builtins.int + resource_exhausted_strategy: builtins.str + """"ignore", "waiting", "throw_error" """ + def __init__( + self, + *, + timeout_seconds: builtins.int = ..., + resource_exhausted_strategy: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["resource_exhausted_strategy", b"resource_exhausted_strategy", "timeout_seconds", b"timeout_seconds"]) -> None: ... + +global___ResourceOptions = ResourceOptions + +@typing_extensions.final +class SubmitJobResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + PROCESSORS_FIELD_NUMBER: builtins.int + session_id: builtins.str + @property + def processors(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[meta_pb2.Processor]: ... + def __init__( + self, + *, + session_id: builtins.str = ..., + processors: collections.abc.Iterable[meta_pb2.Processor] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["processors", b"processors", "session_id", b"session_id"]) -> None: ... + +global___SubmitJobResponse = SubmitJobResponse + +@typing_extensions.final +class KillJobRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + session_id: builtins.str + def __init__( + self, + *, + session_id: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["session_id", b"session_id"]) -> None: ... + +global___KillJobRequest = KillJobRequest + +@typing_extensions.final +class KillJobResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + session_id: builtins.str + def __init__( + self, + *, + session_id: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["session_id", b"session_id"]) -> None: ... + +global___KillJobResponse = KillJobResponse + +@typing_extensions.final +class StopJobRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + session_id: builtins.str + def __init__( + self, + *, + session_id: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["session_id", b"session_id"]) -> None: ... + +global___StopJobRequest = StopJobRequest + +@typing_extensions.final +class StopJobResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + session_id: builtins.str + def __init__( + self, + *, + session_id: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["session_id", b"session_id"]) -> None: ... + +global___StopJobResponse = StopJobResponse + +@typing_extensions.final +class QueryJobRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + session_id: builtins.str + def __init__( + self, + *, + session_id: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["session_id", b"session_id"]) -> None: ... + +global___QueryJobRequest = QueryJobRequest + +@typing_extensions.final +class QueryJobResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + JOB_TYPE_FIELD_NUMBER: builtins.int + STATUS_FIELD_NUMBER: builtins.int + PROCESSORS_FIELD_NUMBER: builtins.int + session_id: builtins.str + job_type: builtins.str + status: builtins.str + @property + def processors(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[meta_pb2.Processor]: ... + def __init__( + self, + *, + session_id: builtins.str = ..., + job_type: builtins.str = ..., + status: builtins.str = ..., + processors: collections.abc.Iterable[meta_pb2.Processor] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["job_type", b"job_type", "processors", b"processors", "session_id", b"session_id", "status", b"status"]) -> None: ... + +global___QueryJobResponse = QueryJobResponse + +@typing_extensions.final +class QueryJobStatusRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + session_id: builtins.str + def __init__( + self, + *, + session_id: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["session_id", b"session_id"]) -> None: ... + +global___QueryJobStatusRequest = QueryJobStatusRequest + +@typing_extensions.final +class QueryJobStatusResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + STATUS_FIELD_NUMBER: builtins.int + session_id: builtins.str + status: builtins.str + def __init__( + self, + *, + session_id: builtins.str = ..., + status: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["session_id", b"session_id", "status", b"status"]) -> None: ... + +global___QueryJobStatusResponse = QueryJobStatusResponse + +@typing_extensions.final +class DownloadJobRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + RANKS_FIELD_NUMBER: builtins.int + COMPRESS_METHOD_FIELD_NUMBER: builtins.int + COMPRESS_LEVEL_FIELD_NUMBER: builtins.int + CONTENT_TYPE_FIELD_NUMBER: builtins.int + session_id: builtins.str + @property + def ranks(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ... + compress_method: builtins.str + compress_level: builtins.int + content_type: containers_pb2.ContentType.ValueType + def __init__( + self, + *, + session_id: builtins.str = ..., + ranks: collections.abc.Iterable[builtins.int] | None = ..., + compress_method: builtins.str = ..., + compress_level: builtins.int = ..., + content_type: containers_pb2.ContentType.ValueType = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["compress_level", b"compress_level", "compress_method", b"compress_method", "content_type", b"content_type", "ranks", b"ranks", "session_id", b"session_id"]) -> None: ... + +global___DownloadJobRequest = DownloadJobRequest + +@typing_extensions.final +class DownloadJobResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSION_ID_FIELD_NUMBER: builtins.int + CONTAINER_CONTENT_FIELD_NUMBER: builtins.int + session_id: builtins.str + @property + def container_content(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[containers_pb2.ContainerContent]: ... + def __init__( + self, + *, + session_id: builtins.str = ..., + container_content: collections.abc.Iterable[containers_pb2.ContainerContent] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["container_content", b"container_content", "session_id", b"session_id"]) -> None: ... + +global___DownloadJobResponse = DownloadJobResponse diff --git a/python/eggroll/backport/proto/deepspeed_pb2_grpc.py b/python/eggroll/backport/proto/deepspeed_pb2_grpc.py new file mode 100644 index 000000000..aeaf47ec8 --- /dev/null +++ b/python/eggroll/backport/proto/deepspeed_pb2_grpc.py @@ -0,0 +1,264 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +import deepspeed_pb2 as deepspeed__pb2 + + +class DeepspeedRendezvousStoreStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.Set = channel.unary_unary( + '/com.webank.eggroll.core.meta.DeepspeedRendezvousStore/Set', + request_serializer=deepspeed__pb2.StoreSetRequest.SerializeToString, + response_deserializer=deepspeed__pb2.StoreSetResponse.FromString, + ) + self.Get = channel.unary_unary( + '/com.webank.eggroll.core.meta.DeepspeedRendezvousStore/Get', + request_serializer=deepspeed__pb2.StoreGetRequest.SerializeToString, + response_deserializer=deepspeed__pb2.StoreGetResponse.FromString, + ) + self.Add = channel.unary_unary( + '/com.webank.eggroll.core.meta.DeepspeedRendezvousStore/Add', + request_serializer=deepspeed__pb2.StoreAddRequest.SerializeToString, + response_deserializer=deepspeed__pb2.StoreAddResponse.FromString, + ) + self.CompareAndSet = channel.unary_unary( + '/com.webank.eggroll.core.meta.DeepspeedRendezvousStore/CompareAndSet', + request_serializer=deepspeed__pb2.StoreCompareAndSetRequest.SerializeToString, + response_deserializer=deepspeed__pb2.StoreCompareAndSetResponse.FromString, + ) + self.Wait = channel.unary_unary( + '/com.webank.eggroll.core.meta.DeepspeedRendezvousStore/Wait', + request_serializer=deepspeed__pb2.StoreWaitRequest.SerializeToString, + response_deserializer=deepspeed__pb2.StoreWaitResponse.FromString, + ) + self.NumKeys = channel.unary_unary( + '/com.webank.eggroll.core.meta.DeepspeedRendezvousStore/NumKeys', + request_serializer=deepspeed__pb2.StoreNumKeysRequest.SerializeToString, + response_deserializer=deepspeed__pb2.StoreNumKeysResponse.FromString, + ) + self.DeleteKey = channel.unary_unary( + '/com.webank.eggroll.core.meta.DeepspeedRendezvousStore/DeleteKey', + request_serializer=deepspeed__pb2.StoreDeleteKeyRequest.SerializeToString, + response_deserializer=deepspeed__pb2.StoreDeleteKeyResponse.FromString, + ) + + +class DeepspeedRendezvousStoreServicer(object): + """Missing associated documentation comment in .proto file.""" + + def Set(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Get(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Add(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def CompareAndSet(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Wait(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def NumKeys(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def DeleteKey(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_DeepspeedRendezvousStoreServicer_to_server(servicer, server): + rpc_method_handlers = { + 'Set': grpc.unary_unary_rpc_method_handler( + servicer.Set, + request_deserializer=deepspeed__pb2.StoreSetRequest.FromString, + response_serializer=deepspeed__pb2.StoreSetResponse.SerializeToString, + ), + 'Get': grpc.unary_unary_rpc_method_handler( + servicer.Get, + request_deserializer=deepspeed__pb2.StoreGetRequest.FromString, + response_serializer=deepspeed__pb2.StoreGetResponse.SerializeToString, + ), + 'Add': grpc.unary_unary_rpc_method_handler( + servicer.Add, + request_deserializer=deepspeed__pb2.StoreAddRequest.FromString, + response_serializer=deepspeed__pb2.StoreAddResponse.SerializeToString, + ), + 'CompareAndSet': grpc.unary_unary_rpc_method_handler( + servicer.CompareAndSet, + request_deserializer=deepspeed__pb2.StoreCompareAndSetRequest.FromString, + response_serializer=deepspeed__pb2.StoreCompareAndSetResponse.SerializeToString, + ), + 'Wait': grpc.unary_unary_rpc_method_handler( + servicer.Wait, + request_deserializer=deepspeed__pb2.StoreWaitRequest.FromString, + response_serializer=deepspeed__pb2.StoreWaitResponse.SerializeToString, + ), + 'NumKeys': grpc.unary_unary_rpc_method_handler( + servicer.NumKeys, + request_deserializer=deepspeed__pb2.StoreNumKeysRequest.FromString, + response_serializer=deepspeed__pb2.StoreNumKeysResponse.SerializeToString, + ), + 'DeleteKey': grpc.unary_unary_rpc_method_handler( + servicer.DeleteKey, + request_deserializer=deepspeed__pb2.StoreDeleteKeyRequest.FromString, + response_serializer=deepspeed__pb2.StoreDeleteKeyResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'com.webank.eggroll.core.meta.DeepspeedRendezvousStore', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class DeepspeedRendezvousStore(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def Set(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/com.webank.eggroll.core.meta.DeepspeedRendezvousStore/Set', + deepspeed__pb2.StoreSetRequest.SerializeToString, + deepspeed__pb2.StoreSetResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Get(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/com.webank.eggroll.core.meta.DeepspeedRendezvousStore/Get', + deepspeed__pb2.StoreGetRequest.SerializeToString, + deepspeed__pb2.StoreGetResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Add(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/com.webank.eggroll.core.meta.DeepspeedRendezvousStore/Add', + deepspeed__pb2.StoreAddRequest.SerializeToString, + deepspeed__pb2.StoreAddResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def CompareAndSet(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/com.webank.eggroll.core.meta.DeepspeedRendezvousStore/CompareAndSet', + deepspeed__pb2.StoreCompareAndSetRequest.SerializeToString, + deepspeed__pb2.StoreCompareAndSetResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Wait(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/com.webank.eggroll.core.meta.DeepspeedRendezvousStore/Wait', + deepspeed__pb2.StoreWaitRequest.SerializeToString, + deepspeed__pb2.StoreWaitResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def NumKeys(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/com.webank.eggroll.core.meta.DeepspeedRendezvousStore/NumKeys', + deepspeed__pb2.StoreNumKeysRequest.SerializeToString, + deepspeed__pb2.StoreNumKeysResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def DeleteKey(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/com.webank.eggroll.core.meta.DeepspeedRendezvousStore/DeleteKey', + deepspeed__pb2.StoreDeleteKeyRequest.SerializeToString, + deepspeed__pb2.StoreDeleteKeyResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/python/eggroll/backport/proto/egg_pb2.py b/python/eggroll/backport/proto/egg_pb2.py new file mode 100644 index 000000000..462430a99 --- /dev/null +++ b/python/eggroll/backport/proto/egg_pb2.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: egg.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\tegg.proto\x12\x1b\x63om.webank.eggroll.core.egg\"\x1e\n\rCountResponse\x12\r\n\x05value\x18\x01 \x01(\x03\"\x19\n\nGetRequest\x12\x0b\n\x03key\x18\x01 \x01(\x0c\"9\n\x0bGetResponse\x12\x0b\n\x03key\x18\x01 \x01(\x0c\x12\r\n\x05value\x18\x02 \x01(\x0c\x12\x0e\n\x06\x65xists\x18\x03 \x01(\x08\"(\n\nPutRequest\x12\x0b\n\x03key\x18\x01 \x01(\x0c\x12\r\n\x05value\x18\x02 \x01(\x0c\":\n\x0bPutResponse\x12\x0b\n\x03key\x18\x01 \x01(\x0c\x12\r\n\x05value\x18\x02 \x01(\x0c\x12\x0f\n\x07success\x18\x03 \x01(\x08\"\x1e\n\rGetAllRequest\x12\r\n\x05limit\x18\x01 \x01(\x03\"\x1c\n\rDeleteRequest\x12\x0b\n\x03key\x18\x01 \x01(\x0c\".\n\x0e\x44\x65leteResponse\x12\x0b\n\x03key\x18\x01 \x01(\x0c\x12\x0f\n\x07success\x18\x02 \x01(\x08\"0\n\x1dMapPartitionsWithIndexRequest\x12\x0f\n\x07shuffle\x18\x01 \x01(\x08\"+\n\x0eReduceResponse\x12\n\n\x02id\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\x0c\"&\n\x10\x41ggregateRequest\x12\x12\n\nzero_value\x18\x01 \x01(\x0c\".\n\x11\x41ggregateResponse\x12\n\n\x02id\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\x0c\"/\n\x12WithStoresResponse\x12\n\n\x02id\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\x0c\x62\x06proto3') + + + +_COUNTRESPONSE = DESCRIPTOR.message_types_by_name['CountResponse'] +_GETREQUEST = DESCRIPTOR.message_types_by_name['GetRequest'] +_GETRESPONSE = DESCRIPTOR.message_types_by_name['GetResponse'] +_PUTREQUEST = DESCRIPTOR.message_types_by_name['PutRequest'] +_PUTRESPONSE = DESCRIPTOR.message_types_by_name['PutResponse'] +_GETALLREQUEST = DESCRIPTOR.message_types_by_name['GetAllRequest'] +_DELETEREQUEST = DESCRIPTOR.message_types_by_name['DeleteRequest'] +_DELETERESPONSE = DESCRIPTOR.message_types_by_name['DeleteResponse'] +_MAPPARTITIONSWITHINDEXREQUEST = DESCRIPTOR.message_types_by_name['MapPartitionsWithIndexRequest'] +_REDUCERESPONSE = DESCRIPTOR.message_types_by_name['ReduceResponse'] +_AGGREGATEREQUEST = DESCRIPTOR.message_types_by_name['AggregateRequest'] +_AGGREGATERESPONSE = DESCRIPTOR.message_types_by_name['AggregateResponse'] +_WITHSTORESRESPONSE = DESCRIPTOR.message_types_by_name['WithStoresResponse'] +CountResponse = _reflection.GeneratedProtocolMessageType('CountResponse', (_message.Message,), { + 'DESCRIPTOR' : _COUNTRESPONSE, + '__module__' : 'egg_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.egg.CountResponse) + }) +_sym_db.RegisterMessage(CountResponse) + +GetRequest = _reflection.GeneratedProtocolMessageType('GetRequest', (_message.Message,), { + 'DESCRIPTOR' : _GETREQUEST, + '__module__' : 'egg_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.egg.GetRequest) + }) +_sym_db.RegisterMessage(GetRequest) + +GetResponse = _reflection.GeneratedProtocolMessageType('GetResponse', (_message.Message,), { + 'DESCRIPTOR' : _GETRESPONSE, + '__module__' : 'egg_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.egg.GetResponse) + }) +_sym_db.RegisterMessage(GetResponse) + +PutRequest = _reflection.GeneratedProtocolMessageType('PutRequest', (_message.Message,), { + 'DESCRIPTOR' : _PUTREQUEST, + '__module__' : 'egg_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.egg.PutRequest) + }) +_sym_db.RegisterMessage(PutRequest) + +PutResponse = _reflection.GeneratedProtocolMessageType('PutResponse', (_message.Message,), { + 'DESCRIPTOR' : _PUTRESPONSE, + '__module__' : 'egg_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.egg.PutResponse) + }) +_sym_db.RegisterMessage(PutResponse) + +GetAllRequest = _reflection.GeneratedProtocolMessageType('GetAllRequest', (_message.Message,), { + 'DESCRIPTOR' : _GETALLREQUEST, + '__module__' : 'egg_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.egg.GetAllRequest) + }) +_sym_db.RegisterMessage(GetAllRequest) + +DeleteRequest = _reflection.GeneratedProtocolMessageType('DeleteRequest', (_message.Message,), { + 'DESCRIPTOR' : _DELETEREQUEST, + '__module__' : 'egg_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.egg.DeleteRequest) + }) +_sym_db.RegisterMessage(DeleteRequest) + +DeleteResponse = _reflection.GeneratedProtocolMessageType('DeleteResponse', (_message.Message,), { + 'DESCRIPTOR' : _DELETERESPONSE, + '__module__' : 'egg_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.egg.DeleteResponse) + }) +_sym_db.RegisterMessage(DeleteResponse) + +MapPartitionsWithIndexRequest = _reflection.GeneratedProtocolMessageType('MapPartitionsWithIndexRequest', (_message.Message,), { + 'DESCRIPTOR' : _MAPPARTITIONSWITHINDEXREQUEST, + '__module__' : 'egg_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.egg.MapPartitionsWithIndexRequest) + }) +_sym_db.RegisterMessage(MapPartitionsWithIndexRequest) + +ReduceResponse = _reflection.GeneratedProtocolMessageType('ReduceResponse', (_message.Message,), { + 'DESCRIPTOR' : _REDUCERESPONSE, + '__module__' : 'egg_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.egg.ReduceResponse) + }) +_sym_db.RegisterMessage(ReduceResponse) + +AggregateRequest = _reflection.GeneratedProtocolMessageType('AggregateRequest', (_message.Message,), { + 'DESCRIPTOR' : _AGGREGATEREQUEST, + '__module__' : 'egg_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.egg.AggregateRequest) + }) +_sym_db.RegisterMessage(AggregateRequest) + +AggregateResponse = _reflection.GeneratedProtocolMessageType('AggregateResponse', (_message.Message,), { + 'DESCRIPTOR' : _AGGREGATERESPONSE, + '__module__' : 'egg_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.egg.AggregateResponse) + }) +_sym_db.RegisterMessage(AggregateResponse) + +WithStoresResponse = _reflection.GeneratedProtocolMessageType('WithStoresResponse', (_message.Message,), { + 'DESCRIPTOR' : _WITHSTORESRESPONSE, + '__module__' : 'egg_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.egg.WithStoresResponse) + }) +_sym_db.RegisterMessage(WithStoresResponse) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _COUNTRESPONSE._serialized_start=42 + _COUNTRESPONSE._serialized_end=72 + _GETREQUEST._serialized_start=74 + _GETREQUEST._serialized_end=99 + _GETRESPONSE._serialized_start=101 + _GETRESPONSE._serialized_end=158 + _PUTREQUEST._serialized_start=160 + _PUTREQUEST._serialized_end=200 + _PUTRESPONSE._serialized_start=202 + _PUTRESPONSE._serialized_end=260 + _GETALLREQUEST._serialized_start=262 + _GETALLREQUEST._serialized_end=292 + _DELETEREQUEST._serialized_start=294 + _DELETEREQUEST._serialized_end=322 + _DELETERESPONSE._serialized_start=324 + _DELETERESPONSE._serialized_end=370 + _MAPPARTITIONSWITHINDEXREQUEST._serialized_start=372 + _MAPPARTITIONSWITHINDEXREQUEST._serialized_end=420 + _REDUCERESPONSE._serialized_start=422 + _REDUCERESPONSE._serialized_end=465 + _AGGREGATEREQUEST._serialized_start=467 + _AGGREGATEREQUEST._serialized_end=505 + _AGGREGATERESPONSE._serialized_start=507 + _AGGREGATERESPONSE._serialized_end=553 + _WITHSTORESRESPONSE._serialized_start=555 + _WITHSTORESRESPONSE._serialized_end=602 +# @@protoc_insertion_point(module_scope) diff --git a/python/eggroll/backport/proto/egg_pb2.pyi b/python/eggroll/backport/proto/egg_pb2.pyi new file mode 100644 index 000000000..a841318c9 --- /dev/null +++ b/python/eggroll/backport/proto/egg_pb2.pyi @@ -0,0 +1,251 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file + +Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import google.protobuf.descriptor +import google.protobuf.message +import sys + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing_extensions.final +class CountResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + VALUE_FIELD_NUMBER: builtins.int + value: builtins.int + def __init__( + self, + *, + value: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["value", b"value"]) -> None: ... + +global___CountResponse = CountResponse + +@typing_extensions.final +class GetRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + key: builtins.bytes + def __init__( + self, + *, + key: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key"]) -> None: ... + +global___GetRequest = GetRequest + +@typing_extensions.final +class GetResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + EXISTS_FIELD_NUMBER: builtins.int + key: builtins.bytes + value: builtins.bytes + exists: builtins.bool + def __init__( + self, + *, + key: builtins.bytes = ..., + value: builtins.bytes = ..., + exists: builtins.bool = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["exists", b"exists", "key", b"key", "value", b"value"]) -> None: ... + +global___GetResponse = GetResponse + +@typing_extensions.final +class PutRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.bytes + value: builtins.bytes + def __init__( + self, + *, + key: builtins.bytes = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + +global___PutRequest = PutRequest + +@typing_extensions.final +class PutResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + SUCCESS_FIELD_NUMBER: builtins.int + key: builtins.bytes + value: builtins.bytes + success: builtins.bool + def __init__( + self, + *, + key: builtins.bytes = ..., + value: builtins.bytes = ..., + success: builtins.bool = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "success", b"success", "value", b"value"]) -> None: ... + +global___PutResponse = PutResponse + +@typing_extensions.final +class GetAllRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + LIMIT_FIELD_NUMBER: builtins.int + limit: builtins.int + def __init__( + self, + *, + limit: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["limit", b"limit"]) -> None: ... + +global___GetAllRequest = GetAllRequest + +@typing_extensions.final +class DeleteRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + key: builtins.bytes + def __init__( + self, + *, + key: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key"]) -> None: ... + +global___DeleteRequest = DeleteRequest + +@typing_extensions.final +class DeleteResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + SUCCESS_FIELD_NUMBER: builtins.int + key: builtins.bytes + success: builtins.bool + def __init__( + self, + *, + key: builtins.bytes = ..., + success: builtins.bool = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "success", b"success"]) -> None: ... + +global___DeleteResponse = DeleteResponse + +@typing_extensions.final +class MapPartitionsWithIndexRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SHUFFLE_FIELD_NUMBER: builtins.int + shuffle: builtins.bool + def __init__( + self, + *, + shuffle: builtins.bool = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["shuffle", b"shuffle"]) -> None: ... + +global___MapPartitionsWithIndexRequest = MapPartitionsWithIndexRequest + +@typing_extensions.final +class ReduceResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ID_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + id: builtins.int + value: builtins.bytes + def __init__( + self, + *, + id: builtins.int = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["id", b"id", "value", b"value"]) -> None: ... + +global___ReduceResponse = ReduceResponse + +@typing_extensions.final +class AggregateRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ZERO_VALUE_FIELD_NUMBER: builtins.int + zero_value: builtins.bytes + def __init__( + self, + *, + zero_value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["zero_value", b"zero_value"]) -> None: ... + +global___AggregateRequest = AggregateRequest + +@typing_extensions.final +class AggregateResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ID_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + id: builtins.int + value: builtins.bytes + def __init__( + self, + *, + id: builtins.int = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["id", b"id", "value", b"value"]) -> None: ... + +global___AggregateResponse = AggregateResponse + +@typing_extensions.final +class WithStoresResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ID_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + id: builtins.int + value: builtins.bytes + def __init__( + self, + *, + id: builtins.int = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["id", b"id", "value", b"value"]) -> None: ... + +global___WithStoresResponse = WithStoresResponse diff --git a/python/eggroll/backport/proto/extend_pb2.py b/python/eggroll/backport/proto/extend_pb2.py new file mode 100644 index 000000000..93116c383 --- /dev/null +++ b/python/eggroll/backport/proto/extend_pb2.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: extend.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x65xtend.proto\x12 com.webank.eggroll.core.transfer\"b\n\rGetLogRequest\x12\x11\n\tsessionId\x18\x01 \x01(\t\x12\x0c\n\x04rank\x18\x02 \x01(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x11\n\tstartLine\x18\x04 \x01(\x05\x12\x0f\n\x07logType\x18\x05 \x01(\t\":\n\x0eGetLogResponse\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\r\n\x05\x64\x61tas\x18\x02 \x03(\t\x12\x0b\n\x03msg\x18\x03 \x01(\t2\x87\x01\n\x14\x45xtendTransferServer\x12o\n\x06getLog\x12/.com.webank.eggroll.core.transfer.GetLogRequest\x1a\x30.com.webank.eggroll.core.transfer.GetLogResponse(\x01\x30\x01\x62\x06proto3') + + + +_GETLOGREQUEST = DESCRIPTOR.message_types_by_name['GetLogRequest'] +_GETLOGRESPONSE = DESCRIPTOR.message_types_by_name['GetLogResponse'] +GetLogRequest = _reflection.GeneratedProtocolMessageType('GetLogRequest', (_message.Message,), { + 'DESCRIPTOR' : _GETLOGREQUEST, + '__module__' : 'extend_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.transfer.GetLogRequest) + }) +_sym_db.RegisterMessage(GetLogRequest) + +GetLogResponse = _reflection.GeneratedProtocolMessageType('GetLogResponse', (_message.Message,), { + 'DESCRIPTOR' : _GETLOGRESPONSE, + '__module__' : 'extend_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.transfer.GetLogResponse) + }) +_sym_db.RegisterMessage(GetLogResponse) + +_EXTENDTRANSFERSERVER = DESCRIPTOR.services_by_name['ExtendTransferServer'] +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _GETLOGREQUEST._serialized_start=50 + _GETLOGREQUEST._serialized_end=148 + _GETLOGRESPONSE._serialized_start=150 + _GETLOGRESPONSE._serialized_end=208 + _EXTENDTRANSFERSERVER._serialized_start=211 + _EXTENDTRANSFERSERVER._serialized_end=346 +# @@protoc_insertion_point(module_scope) diff --git a/python/eggroll/backport/proto/extend_pb2.pyi b/python/eggroll/backport/proto/extend_pb2.pyi new file mode 100644 index 000000000..8f483d867 --- /dev/null +++ b/python/eggroll/backport/proto/extend_pb2.pyi @@ -0,0 +1,80 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file + +Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import collections.abc +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.message +import sys + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing_extensions.final +class GetLogRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SESSIONID_FIELD_NUMBER: builtins.int + RANK_FIELD_NUMBER: builtins.int + PATH_FIELD_NUMBER: builtins.int + STARTLINE_FIELD_NUMBER: builtins.int + LOGTYPE_FIELD_NUMBER: builtins.int + sessionId: builtins.str + rank: builtins.str + path: builtins.str + startLine: builtins.int + logType: builtins.str + def __init__( + self, + *, + sessionId: builtins.str = ..., + rank: builtins.str = ..., + path: builtins.str = ..., + startLine: builtins.int = ..., + logType: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["logType", b"logType", "path", b"path", "rank", b"rank", "sessionId", b"sessionId", "startLine", b"startLine"]) -> None: ... + +global___GetLogRequest = GetLogRequest + +@typing_extensions.final +class GetLogResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CODE_FIELD_NUMBER: builtins.int + DATAS_FIELD_NUMBER: builtins.int + MSG_FIELD_NUMBER: builtins.int + code: builtins.str + @property + def datas(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ... + msg: builtins.str + def __init__( + self, + *, + code: builtins.str = ..., + datas: collections.abc.Iterable[builtins.str] | None = ..., + msg: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["code", b"code", "datas", b"datas", "msg", b"msg"]) -> None: ... + +global___GetLogResponse = GetLogResponse diff --git a/python/eggroll/backport/proto/extend_pb2_grpc.py b/python/eggroll/backport/proto/extend_pb2_grpc.py new file mode 100644 index 000000000..7dfc21caa --- /dev/null +++ b/python/eggroll/backport/proto/extend_pb2_grpc.py @@ -0,0 +1,69 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +import extend_pb2 as extend__pb2 + + +class ExtendTransferServerStub(object): + """TODO: use transfer lib + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.getLog = channel.stream_stream( + '/com.webank.eggroll.core.transfer.ExtendTransferServer/getLog', + request_serializer=extend__pb2.GetLogRequest.SerializeToString, + response_deserializer=extend__pb2.GetLogResponse.FromString, + ) + + +class ExtendTransferServerServicer(object): + """TODO: use transfer lib + """ + + def getLog(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_ExtendTransferServerServicer_to_server(servicer, server): + rpc_method_handlers = { + 'getLog': grpc.stream_stream_rpc_method_handler( + servicer.getLog, + request_deserializer=extend__pb2.GetLogRequest.FromString, + response_serializer=extend__pb2.GetLogResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'com.webank.eggroll.core.transfer.ExtendTransferServer', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class ExtendTransferServer(object): + """TODO: use transfer lib + """ + + @staticmethod + def getLog(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_stream(request_iterator, target, '/com.webank.eggroll.core.transfer.ExtendTransferServer/getLog', + extend__pb2.GetLogRequest.SerializeToString, + extend__pb2.GetLogResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/python/eggroll/backport/proto/meta_pb2.py b/python/eggroll/backport/proto/meta_pb2.py new file mode 100644 index 000000000..9e6e91bc8 --- /dev/null +++ b/python/eggroll/backport/proto/meta_pb2.py @@ -0,0 +1,402 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: meta.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\nmeta.proto\x12\x1c\x63om.webank.eggroll.core.meta\"&\n\x08\x45ndpoint\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\xd0\x01\n\nServerNode\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclusterId\x18\x03 \x01(\x03\x12\x38\n\x08\x65ndpoint\x18\x04 \x01(\x0b\x32&.com.webank.eggroll.core.meta.Endpoint\x12\x10\n\x08nodeType\x18\x05 \x01(\t\x12\x0e\n\x06status\x18\x06 \x01(\t\x12\x39\n\tresources\x18\x07 \x03(\x0b\x32&.com.webank.eggroll.core.meta.Resource\"u\n\rServerCluster\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0c\n\x04name\x18\x02 \x01(\t\x12=\n\x0bserverNodes\x18\x03 \x03(\x0b\x32(.com.webank.eggroll.core.meta.ServerNode\x12\x0b\n\x03tag\x18\x04 \x01(\t\"\xf6\x02\n\tProcessor\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x14\n\x0cserverNodeId\x18\x02 \x01(\x03\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x15\n\rprocessorType\x18\x04 \x01(\t\x12\x0e\n\x06status\x18\x05 \x01(\t\x12?\n\x0f\x63ommandEndpoint\x18\x06 \x01(\x0b\x32&.com.webank.eggroll.core.meta.Endpoint\x12@\n\x10transferEndpoint\x18\x07 \x01(\x0b\x32&.com.webank.eggroll.core.meta.Endpoint\x12\x0b\n\x03pid\x18\t \x01(\x05\x12\x45\n\x07options\x18\x08 \x03(\x0b\x32\x34.com.webank.eggroll.core.meta.Processor.OptionsEntry\x12\x0b\n\x03tag\x18\n \x01(\t\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"t\n\x0eProcessorBatch\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0c\n\x04name\x18\x02 \x01(\t\x12;\n\nprocessors\x18\x03 \x03(\x0b\x32\'.com.webank.eggroll.core.meta.Processor\x12\x0b\n\x03tag\x18\x04 \x01(\t\"\xaa\x01\n\x07\x46unctor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06serdes\x18\x02 \x01(\t\x12\x0c\n\x04\x62ody\x18\x03 \x01(\x0c\x12\x43\n\x07options\x18\x04 \x03(\x0b\x32\x32.com.webank.eggroll.core.meta.Functor.OptionsEntry\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\")\n\x0bPartitioner\x12\x0c\n\x04type\x18\x01 \x01(\x05\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\"$\n\x06Serdes\x12\x0c\n\x04type\x18\x01 \x01(\x05\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\"\"\n\x04Pair\x12\x0b\n\x03key\x18\x01 \x01(\x0c\x12\r\n\x05value\x18\x02 \x01(\x0c\">\n\tPairBatch\x12\x31\n\x05pairs\x18\x01 \x03(\x0b\x32\".com.webank.eggroll.core.meta.Pair\"\xc5\x01\n\x0cStoreLocator\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x12\n\nstore_type\x18\x02 \x01(\t\x12\x11\n\tnamespace\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\x0c\n\x04path\x18\x05 \x01(\t\x12\x18\n\x10total_partitions\x18\x06 \x01(\x05\x12\x17\n\x0fkey_serdes_type\x18\x07 \x01(\x05\x12\x19\n\x11value_serdes_type\x18\x08 \x01(\x05\x12\x18\n\x10partitioner_type\x18\t \x01(\x05\"\xf9\x01\n\x05Store\x12@\n\x0cstoreLocator\x18\x01 \x01(\x0b\x32*.com.webank.eggroll.core.meta.StoreLocator\x12;\n\npartitions\x18\x02 \x03(\x0b\x32\'.com.webank.eggroll.core.meta.Partition\x12\x41\n\x07options\x18\x03 \x03(\x0b\x32\x30.com.webank.eggroll.core.meta.Store.OptionsEntry\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"@\n\tStoreList\x12\x33\n\x06stores\x18\x01 \x03(\x0b\x32#.com.webank.eggroll.core.meta.Store\"\xbf\x01\n\tPartition\x12\n\n\x02id\x18\x01 \x01(\x05\x12@\n\x0cstoreLocator\x18\x02 \x01(\x0b\x32*.com.webank.eggroll.core.meta.StoreLocator\x12:\n\tprocessor\x18\x03 \x01(\x0b\x32\'.com.webank.eggroll.core.meta.Processor\x12\x14\n\x0cserverNodeId\x18\x04 \x01(\x03\x12\x12\n\nrankInNode\x18\x05 \x01(\x05\"\x1b\n\x08\x43\x61llInfo\x12\x0f\n\x07\x63\x61llSeq\x18\x01 \x01(\t\"\xf1\x01\n\x05JobIO\x12\x32\n\x05store\x18\x01 \x01(\x0b\x32#.com.webank.eggroll.core.meta.Store\x12\x38\n\nkey_serdes\x18\x02 \x01(\x0b\x32$.com.webank.eggroll.core.meta.Serdes\x12:\n\x0cvalue_serdes\x18\x03 \x01(\x0b\x32$.com.webank.eggroll.core.meta.Serdes\x12>\n\x0bpartitioner\x18\x04 \x01(\x0b\x32).com.webank.eggroll.core.meta.Partitioner\"\xb4\x02\n\x03Job\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x33\n\x06inputs\x18\x03 \x03(\x0b\x32#.com.webank.eggroll.core.meta.JobIO\x12\x34\n\x07outputs\x18\x04 \x03(\x0b\x32#.com.webank.eggroll.core.meta.JobIO\x12\x37\n\x08\x66unctors\x18\x05 \x03(\x0b\x32%.com.webank.eggroll.core.meta.Functor\x12?\n\x07options\x18\x06 \x03(\x0b\x32..com.webank.eggroll.core.meta.Job.OptionsEntry\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xc3\x01\n\x04Task\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x37\n\x06inputs\x18\x03 \x03(\x0b\x32\'.com.webank.eggroll.core.meta.Partition\x12\x38\n\x07outputs\x18\x04 \x03(\x0b\x32\'.com.webank.eggroll.core.meta.Partition\x12.\n\x03job\x18\x05 \x01(\x0b\x32!.com.webank.eggroll.core.meta.Job\"\xfa\x01\n\x0bSessionMeta\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0e\n\x06status\x18\x03 \x01(\t\x12\x0b\n\x03tag\x18\x04 \x01(\t\x12;\n\nprocessors\x18\x05 \x03(\x0b\x32\'.com.webank.eggroll.core.meta.Processor\x12G\n\x07options\x18\x06 \x03(\x0b\x32\x36.com.webank.eggroll.core.meta.SessionMeta.OptionsEntry\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9d\x01\n\x12ResourceAllocation\x12\x14\n\x0cserverNodeId\x18\x01 \x01(\x03\x12\x0e\n\x06status\x18\x02 \x01(\t\x12\x11\n\tsessionId\x18\x03 \x01(\t\x12\x13\n\x0boperateType\x18\x04 \x01(\t\x12\x39\n\tresources\x18\x05 \x03(\x0b\x32&.com.webank.eggroll.core.meta.Resource\"H\n\x08Resource\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\r\n\x05total\x18\x02 \x01(\x03\x12\x0c\n\x04used\x18\x03 \x01(\x03\x12\x11\n\tallocated\x18\x04 \x01(\x03\"\x9c\x01\n\rNodeHeartbeat\x12\n\n\x02id\x18\x01 \x01(\x04\x12\x36\n\x04node\x18\x02 \x01(\x0b\x32(.com.webank.eggroll.core.meta.ServerNode\x12\x0c\n\x04\x63ode\x18\x03 \x01(\t\x12\x0b\n\x03msg\x18\x04 \x01(\t\x12\x15\n\rgpuProcessors\x18\x05 \x03(\x05\x12\x15\n\rcpuProcessors\x18\x06 \x03(\x05\"\x90\x01\n\x10MetaInfoResponse\x12L\n\x07metaMap\x18\x01 \x03(\x0b\x32;.com.webank.eggroll.core.meta.MetaInfoResponse.MetaMapEntry\x1a.\n\x0cMetaMapEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x1e\n\x0fMetaInfoRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\"&\n\x11QueueViewResponse\x12\x11\n\tqueueSize\x18\x01 \x01(\x05\"\x1f\n\x10QueueViewRequest\x12\x0b\n\x03key\x18\x01 \x01(\t\"h\n\x1a\x43heckResourceEnoughRequest\x12\x15\n\rresource_type\x18\x01 \x01(\t\x12\x1f\n\x17required_resource_count\x18\x02 \x01(\x03\x12\x12\n\ncheck_type\x18\x03 \x01(\t\"r\n\x1b\x43heckResourceEnoughResponse\x12\x11\n\tis_enough\x18\x01 \x01(\x08\x12@\n\x0b\x63lusterInfo\x18\x02 \x01(\x0b\x32+.com.webank.eggroll.core.meta.ServerCluster\"\x19\n\x17queryClusterInfoRequest\"\\\n\x18queryClusterInfoResponse\x12@\n\x0b\x63lusterInfo\x18\x02 \x01(\x0b\x32+.com.webank.eggroll.core.meta.ServerClusterb\x06proto3') + + + +_ENDPOINT = DESCRIPTOR.message_types_by_name['Endpoint'] +_SERVERNODE = DESCRIPTOR.message_types_by_name['ServerNode'] +_SERVERCLUSTER = DESCRIPTOR.message_types_by_name['ServerCluster'] +_PROCESSOR = DESCRIPTOR.message_types_by_name['Processor'] +_PROCESSOR_OPTIONSENTRY = _PROCESSOR.nested_types_by_name['OptionsEntry'] +_PROCESSORBATCH = DESCRIPTOR.message_types_by_name['ProcessorBatch'] +_FUNCTOR = DESCRIPTOR.message_types_by_name['Functor'] +_FUNCTOR_OPTIONSENTRY = _FUNCTOR.nested_types_by_name['OptionsEntry'] +_PARTITIONER = DESCRIPTOR.message_types_by_name['Partitioner'] +_SERDES = DESCRIPTOR.message_types_by_name['Serdes'] +_PAIR = DESCRIPTOR.message_types_by_name['Pair'] +_PAIRBATCH = DESCRIPTOR.message_types_by_name['PairBatch'] +_STORELOCATOR = DESCRIPTOR.message_types_by_name['StoreLocator'] +_STORE = DESCRIPTOR.message_types_by_name['Store'] +_STORE_OPTIONSENTRY = _STORE.nested_types_by_name['OptionsEntry'] +_STORELIST = DESCRIPTOR.message_types_by_name['StoreList'] +_PARTITION = DESCRIPTOR.message_types_by_name['Partition'] +_CALLINFO = DESCRIPTOR.message_types_by_name['CallInfo'] +_JOBIO = DESCRIPTOR.message_types_by_name['JobIO'] +_JOB = DESCRIPTOR.message_types_by_name['Job'] +_JOB_OPTIONSENTRY = _JOB.nested_types_by_name['OptionsEntry'] +_TASK = DESCRIPTOR.message_types_by_name['Task'] +_SESSIONMETA = DESCRIPTOR.message_types_by_name['SessionMeta'] +_SESSIONMETA_OPTIONSENTRY = _SESSIONMETA.nested_types_by_name['OptionsEntry'] +_RESOURCEALLOCATION = DESCRIPTOR.message_types_by_name['ResourceAllocation'] +_RESOURCE = DESCRIPTOR.message_types_by_name['Resource'] +_NODEHEARTBEAT = DESCRIPTOR.message_types_by_name['NodeHeartbeat'] +_METAINFORESPONSE = DESCRIPTOR.message_types_by_name['MetaInfoResponse'] +_METAINFORESPONSE_METAMAPENTRY = _METAINFORESPONSE.nested_types_by_name['MetaMapEntry'] +_METAINFOREQUEST = DESCRIPTOR.message_types_by_name['MetaInfoRequest'] +_QUEUEVIEWRESPONSE = DESCRIPTOR.message_types_by_name['QueueViewResponse'] +_QUEUEVIEWREQUEST = DESCRIPTOR.message_types_by_name['QueueViewRequest'] +_CHECKRESOURCEENOUGHREQUEST = DESCRIPTOR.message_types_by_name['CheckResourceEnoughRequest'] +_CHECKRESOURCEENOUGHRESPONSE = DESCRIPTOR.message_types_by_name['CheckResourceEnoughResponse'] +_QUERYCLUSTERINFOREQUEST = DESCRIPTOR.message_types_by_name['queryClusterInfoRequest'] +_QUERYCLUSTERINFORESPONSE = DESCRIPTOR.message_types_by_name['queryClusterInfoResponse'] +Endpoint = _reflection.GeneratedProtocolMessageType('Endpoint', (_message.Message,), { + 'DESCRIPTOR' : _ENDPOINT, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Endpoint) + }) +_sym_db.RegisterMessage(Endpoint) + +ServerNode = _reflection.GeneratedProtocolMessageType('ServerNode', (_message.Message,), { + 'DESCRIPTOR' : _SERVERNODE, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.ServerNode) + }) +_sym_db.RegisterMessage(ServerNode) + +ServerCluster = _reflection.GeneratedProtocolMessageType('ServerCluster', (_message.Message,), { + 'DESCRIPTOR' : _SERVERCLUSTER, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.ServerCluster) + }) +_sym_db.RegisterMessage(ServerCluster) + +Processor = _reflection.GeneratedProtocolMessageType('Processor', (_message.Message,), { + + 'OptionsEntry' : _reflection.GeneratedProtocolMessageType('OptionsEntry', (_message.Message,), { + 'DESCRIPTOR' : _PROCESSOR_OPTIONSENTRY, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Processor.OptionsEntry) + }) + , + 'DESCRIPTOR' : _PROCESSOR, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Processor) + }) +_sym_db.RegisterMessage(Processor) +_sym_db.RegisterMessage(Processor.OptionsEntry) + +ProcessorBatch = _reflection.GeneratedProtocolMessageType('ProcessorBatch', (_message.Message,), { + 'DESCRIPTOR' : _PROCESSORBATCH, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.ProcessorBatch) + }) +_sym_db.RegisterMessage(ProcessorBatch) + +Functor = _reflection.GeneratedProtocolMessageType('Functor', (_message.Message,), { + + 'OptionsEntry' : _reflection.GeneratedProtocolMessageType('OptionsEntry', (_message.Message,), { + 'DESCRIPTOR' : _FUNCTOR_OPTIONSENTRY, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Functor.OptionsEntry) + }) + , + 'DESCRIPTOR' : _FUNCTOR, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Functor) + }) +_sym_db.RegisterMessage(Functor) +_sym_db.RegisterMessage(Functor.OptionsEntry) + +Partitioner = _reflection.GeneratedProtocolMessageType('Partitioner', (_message.Message,), { + 'DESCRIPTOR' : _PARTITIONER, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Partitioner) + }) +_sym_db.RegisterMessage(Partitioner) + +Serdes = _reflection.GeneratedProtocolMessageType('Serdes', (_message.Message,), { + 'DESCRIPTOR' : _SERDES, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Serdes) + }) +_sym_db.RegisterMessage(Serdes) + +Pair = _reflection.GeneratedProtocolMessageType('Pair', (_message.Message,), { + 'DESCRIPTOR' : _PAIR, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Pair) + }) +_sym_db.RegisterMessage(Pair) + +PairBatch = _reflection.GeneratedProtocolMessageType('PairBatch', (_message.Message,), { + 'DESCRIPTOR' : _PAIRBATCH, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.PairBatch) + }) +_sym_db.RegisterMessage(PairBatch) + +StoreLocator = _reflection.GeneratedProtocolMessageType('StoreLocator', (_message.Message,), { + 'DESCRIPTOR' : _STORELOCATOR, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreLocator) + }) +_sym_db.RegisterMessage(StoreLocator) + +Store = _reflection.GeneratedProtocolMessageType('Store', (_message.Message,), { + + 'OptionsEntry' : _reflection.GeneratedProtocolMessageType('OptionsEntry', (_message.Message,), { + 'DESCRIPTOR' : _STORE_OPTIONSENTRY, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Store.OptionsEntry) + }) + , + 'DESCRIPTOR' : _STORE, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Store) + }) +_sym_db.RegisterMessage(Store) +_sym_db.RegisterMessage(Store.OptionsEntry) + +StoreList = _reflection.GeneratedProtocolMessageType('StoreList', (_message.Message,), { + 'DESCRIPTOR' : _STORELIST, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.StoreList) + }) +_sym_db.RegisterMessage(StoreList) + +Partition = _reflection.GeneratedProtocolMessageType('Partition', (_message.Message,), { + 'DESCRIPTOR' : _PARTITION, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Partition) + }) +_sym_db.RegisterMessage(Partition) + +CallInfo = _reflection.GeneratedProtocolMessageType('CallInfo', (_message.Message,), { + 'DESCRIPTOR' : _CALLINFO, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.CallInfo) + }) +_sym_db.RegisterMessage(CallInfo) + +JobIO = _reflection.GeneratedProtocolMessageType('JobIO', (_message.Message,), { + 'DESCRIPTOR' : _JOBIO, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.JobIO) + }) +_sym_db.RegisterMessage(JobIO) + +Job = _reflection.GeneratedProtocolMessageType('Job', (_message.Message,), { + + 'OptionsEntry' : _reflection.GeneratedProtocolMessageType('OptionsEntry', (_message.Message,), { + 'DESCRIPTOR' : _JOB_OPTIONSENTRY, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Job.OptionsEntry) + }) + , + 'DESCRIPTOR' : _JOB, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Job) + }) +_sym_db.RegisterMessage(Job) +_sym_db.RegisterMessage(Job.OptionsEntry) + +Task = _reflection.GeneratedProtocolMessageType('Task', (_message.Message,), { + 'DESCRIPTOR' : _TASK, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Task) + }) +_sym_db.RegisterMessage(Task) + +SessionMeta = _reflection.GeneratedProtocolMessageType('SessionMeta', (_message.Message,), { + + 'OptionsEntry' : _reflection.GeneratedProtocolMessageType('OptionsEntry', (_message.Message,), { + 'DESCRIPTOR' : _SESSIONMETA_OPTIONSENTRY, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.SessionMeta.OptionsEntry) + }) + , + 'DESCRIPTOR' : _SESSIONMETA, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.SessionMeta) + }) +_sym_db.RegisterMessage(SessionMeta) +_sym_db.RegisterMessage(SessionMeta.OptionsEntry) + +ResourceAllocation = _reflection.GeneratedProtocolMessageType('ResourceAllocation', (_message.Message,), { + 'DESCRIPTOR' : _RESOURCEALLOCATION, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.ResourceAllocation) + }) +_sym_db.RegisterMessage(ResourceAllocation) + +Resource = _reflection.GeneratedProtocolMessageType('Resource', (_message.Message,), { + 'DESCRIPTOR' : _RESOURCE, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.Resource) + }) +_sym_db.RegisterMessage(Resource) + +NodeHeartbeat = _reflection.GeneratedProtocolMessageType('NodeHeartbeat', (_message.Message,), { + 'DESCRIPTOR' : _NODEHEARTBEAT, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.NodeHeartbeat) + }) +_sym_db.RegisterMessage(NodeHeartbeat) + +MetaInfoResponse = _reflection.GeneratedProtocolMessageType('MetaInfoResponse', (_message.Message,), { + + 'MetaMapEntry' : _reflection.GeneratedProtocolMessageType('MetaMapEntry', (_message.Message,), { + 'DESCRIPTOR' : _METAINFORESPONSE_METAMAPENTRY, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.MetaInfoResponse.MetaMapEntry) + }) + , + 'DESCRIPTOR' : _METAINFORESPONSE, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.MetaInfoResponse) + }) +_sym_db.RegisterMessage(MetaInfoResponse) +_sym_db.RegisterMessage(MetaInfoResponse.MetaMapEntry) + +MetaInfoRequest = _reflection.GeneratedProtocolMessageType('MetaInfoRequest', (_message.Message,), { + 'DESCRIPTOR' : _METAINFOREQUEST, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.MetaInfoRequest) + }) +_sym_db.RegisterMessage(MetaInfoRequest) + +QueueViewResponse = _reflection.GeneratedProtocolMessageType('QueueViewResponse', (_message.Message,), { + 'DESCRIPTOR' : _QUEUEVIEWRESPONSE, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.QueueViewResponse) + }) +_sym_db.RegisterMessage(QueueViewResponse) + +QueueViewRequest = _reflection.GeneratedProtocolMessageType('QueueViewRequest', (_message.Message,), { + 'DESCRIPTOR' : _QUEUEVIEWREQUEST, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.QueueViewRequest) + }) +_sym_db.RegisterMessage(QueueViewRequest) + +CheckResourceEnoughRequest = _reflection.GeneratedProtocolMessageType('CheckResourceEnoughRequest', (_message.Message,), { + 'DESCRIPTOR' : _CHECKRESOURCEENOUGHREQUEST, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.CheckResourceEnoughRequest) + }) +_sym_db.RegisterMessage(CheckResourceEnoughRequest) + +CheckResourceEnoughResponse = _reflection.GeneratedProtocolMessageType('CheckResourceEnoughResponse', (_message.Message,), { + 'DESCRIPTOR' : _CHECKRESOURCEENOUGHRESPONSE, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.CheckResourceEnoughResponse) + }) +_sym_db.RegisterMessage(CheckResourceEnoughResponse) + +queryClusterInfoRequest = _reflection.GeneratedProtocolMessageType('queryClusterInfoRequest', (_message.Message,), { + 'DESCRIPTOR' : _QUERYCLUSTERINFOREQUEST, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.queryClusterInfoRequest) + }) +_sym_db.RegisterMessage(queryClusterInfoRequest) + +queryClusterInfoResponse = _reflection.GeneratedProtocolMessageType('queryClusterInfoResponse', (_message.Message,), { + 'DESCRIPTOR' : _QUERYCLUSTERINFORESPONSE, + '__module__' : 'meta_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.meta.queryClusterInfoResponse) + }) +_sym_db.RegisterMessage(queryClusterInfoResponse) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _PROCESSOR_OPTIONSENTRY._options = None + _PROCESSOR_OPTIONSENTRY._serialized_options = b'8\001' + _FUNCTOR_OPTIONSENTRY._options = None + _FUNCTOR_OPTIONSENTRY._serialized_options = b'8\001' + _STORE_OPTIONSENTRY._options = None + _STORE_OPTIONSENTRY._serialized_options = b'8\001' + _JOB_OPTIONSENTRY._options = None + _JOB_OPTIONSENTRY._serialized_options = b'8\001' + _SESSIONMETA_OPTIONSENTRY._options = None + _SESSIONMETA_OPTIONSENTRY._serialized_options = b'8\001' + _METAINFORESPONSE_METAMAPENTRY._options = None + _METAINFORESPONSE_METAMAPENTRY._serialized_options = b'8\001' + _ENDPOINT._serialized_start=44 + _ENDPOINT._serialized_end=82 + _SERVERNODE._serialized_start=85 + _SERVERNODE._serialized_end=293 + _SERVERCLUSTER._serialized_start=295 + _SERVERCLUSTER._serialized_end=412 + _PROCESSOR._serialized_start=415 + _PROCESSOR._serialized_end=789 + _PROCESSOR_OPTIONSENTRY._serialized_start=743 + _PROCESSOR_OPTIONSENTRY._serialized_end=789 + _PROCESSORBATCH._serialized_start=791 + _PROCESSORBATCH._serialized_end=907 + _FUNCTOR._serialized_start=910 + _FUNCTOR._serialized_end=1080 + _FUNCTOR_OPTIONSENTRY._serialized_start=743 + _FUNCTOR_OPTIONSENTRY._serialized_end=789 + _PARTITIONER._serialized_start=1082 + _PARTITIONER._serialized_end=1123 + _SERDES._serialized_start=1125 + _SERDES._serialized_end=1161 + _PAIR._serialized_start=1163 + _PAIR._serialized_end=1197 + _PAIRBATCH._serialized_start=1199 + _PAIRBATCH._serialized_end=1261 + _STORELOCATOR._serialized_start=1264 + _STORELOCATOR._serialized_end=1461 + _STORE._serialized_start=1464 + _STORE._serialized_end=1713 + _STORE_OPTIONSENTRY._serialized_start=743 + _STORE_OPTIONSENTRY._serialized_end=789 + _STORELIST._serialized_start=1715 + _STORELIST._serialized_end=1779 + _PARTITION._serialized_start=1782 + _PARTITION._serialized_end=1973 + _CALLINFO._serialized_start=1975 + _CALLINFO._serialized_end=2002 + _JOBIO._serialized_start=2005 + _JOBIO._serialized_end=2246 + _JOB._serialized_start=2249 + _JOB._serialized_end=2557 + _JOB_OPTIONSENTRY._serialized_start=743 + _JOB_OPTIONSENTRY._serialized_end=789 + _TASK._serialized_start=2560 + _TASK._serialized_end=2755 + _SESSIONMETA._serialized_start=2758 + _SESSIONMETA._serialized_end=3008 + _SESSIONMETA_OPTIONSENTRY._serialized_start=743 + _SESSIONMETA_OPTIONSENTRY._serialized_end=789 + _RESOURCEALLOCATION._serialized_start=3011 + _RESOURCEALLOCATION._serialized_end=3168 + _RESOURCE._serialized_start=3170 + _RESOURCE._serialized_end=3242 + _NODEHEARTBEAT._serialized_start=3245 + _NODEHEARTBEAT._serialized_end=3401 + _METAINFORESPONSE._serialized_start=3404 + _METAINFORESPONSE._serialized_end=3548 + _METAINFORESPONSE_METAMAPENTRY._serialized_start=3502 + _METAINFORESPONSE_METAMAPENTRY._serialized_end=3548 + _METAINFOREQUEST._serialized_start=3550 + _METAINFOREQUEST._serialized_end=3580 + _QUEUEVIEWRESPONSE._serialized_start=3582 + _QUEUEVIEWRESPONSE._serialized_end=3620 + _QUEUEVIEWREQUEST._serialized_start=3622 + _QUEUEVIEWREQUEST._serialized_end=3653 + _CHECKRESOURCEENOUGHREQUEST._serialized_start=3655 + _CHECKRESOURCEENOUGHREQUEST._serialized_end=3759 + _CHECKRESOURCEENOUGHRESPONSE._serialized_start=3761 + _CHECKRESOURCEENOUGHRESPONSE._serialized_end=3875 + _QUERYCLUSTERINFOREQUEST._serialized_start=3877 + _QUERYCLUSTERINFOREQUEST._serialized_end=3902 + _QUERYCLUSTERINFORESPONSE._serialized_start=3904 + _QUERYCLUSTERINFORESPONSE._serialized_end=3996 +# @@protoc_insertion_point(module_scope) diff --git a/python/eggroll/backport/proto/meta_pb2.pyi b/python/eggroll/backport/proto/meta_pb2.pyi new file mode 100644 index 000000000..4f827c882 --- /dev/null +++ b/python/eggroll/backport/proto/meta_pb2.pyi @@ -0,0 +1,840 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file + +Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import collections.abc +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.message +import sys + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing_extensions.final +class Endpoint(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + HOST_FIELD_NUMBER: builtins.int + PORT_FIELD_NUMBER: builtins.int + host: builtins.str + port: builtins.int + def __init__( + self, + *, + host: builtins.str = ..., + port: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["host", b"host", "port", b"port"]) -> None: ... + +global___Endpoint = Endpoint + +@typing_extensions.final +class ServerNode(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ID_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + CLUSTERID_FIELD_NUMBER: builtins.int + ENDPOINT_FIELD_NUMBER: builtins.int + NODETYPE_FIELD_NUMBER: builtins.int + STATUS_FIELD_NUMBER: builtins.int + RESOURCES_FIELD_NUMBER: builtins.int + id: builtins.int + name: builtins.str + clusterId: builtins.int + @property + def endpoint(self) -> global___Endpoint: ... + nodeType: builtins.str + status: builtins.str + @property + def resources(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Resource]: ... + def __init__( + self, + *, + id: builtins.int = ..., + name: builtins.str = ..., + clusterId: builtins.int = ..., + endpoint: global___Endpoint | None = ..., + nodeType: builtins.str = ..., + status: builtins.str = ..., + resources: collections.abc.Iterable[global___Resource] | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["endpoint", b"endpoint"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["clusterId", b"clusterId", "endpoint", b"endpoint", "id", b"id", "name", b"name", "nodeType", b"nodeType", "resources", b"resources", "status", b"status"]) -> None: ... + +global___ServerNode = ServerNode + +@typing_extensions.final +class ServerCluster(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ID_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + SERVERNODES_FIELD_NUMBER: builtins.int + TAG_FIELD_NUMBER: builtins.int + id: builtins.int + name: builtins.str + @property + def serverNodes(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___ServerNode]: ... + tag: builtins.str + def __init__( + self, + *, + id: builtins.int = ..., + name: builtins.str = ..., + serverNodes: collections.abc.Iterable[global___ServerNode] | None = ..., + tag: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["id", b"id", "name", b"name", "serverNodes", b"serverNodes", "tag", b"tag"]) -> None: ... + +global___ServerCluster = ServerCluster + +@typing_extensions.final +class Processor(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class OptionsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + ID_FIELD_NUMBER: builtins.int + SERVERNODEID_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + PROCESSORTYPE_FIELD_NUMBER: builtins.int + STATUS_FIELD_NUMBER: builtins.int + COMMANDENDPOINT_FIELD_NUMBER: builtins.int + TRANSFERENDPOINT_FIELD_NUMBER: builtins.int + PID_FIELD_NUMBER: builtins.int + OPTIONS_FIELD_NUMBER: builtins.int + TAG_FIELD_NUMBER: builtins.int + id: builtins.int + serverNodeId: builtins.int + name: builtins.str + processorType: builtins.str + status: builtins.str + @property + def commandEndpoint(self) -> global___Endpoint: ... + @property + def transferEndpoint(self) -> global___Endpoint: ... + pid: builtins.int + @property + def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ... + tag: builtins.str + def __init__( + self, + *, + id: builtins.int = ..., + serverNodeId: builtins.int = ..., + name: builtins.str = ..., + processorType: builtins.str = ..., + status: builtins.str = ..., + commandEndpoint: global___Endpoint | None = ..., + transferEndpoint: global___Endpoint | None = ..., + pid: builtins.int = ..., + options: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + tag: builtins.str = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["commandEndpoint", b"commandEndpoint", "transferEndpoint", b"transferEndpoint"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["commandEndpoint", b"commandEndpoint", "id", b"id", "name", b"name", "options", b"options", "pid", b"pid", "processorType", b"processorType", "serverNodeId", b"serverNodeId", "status", b"status", "tag", b"tag", "transferEndpoint", b"transferEndpoint"]) -> None: ... + +global___Processor = Processor + +@typing_extensions.final +class ProcessorBatch(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ID_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + PROCESSORS_FIELD_NUMBER: builtins.int + TAG_FIELD_NUMBER: builtins.int + id: builtins.int + name: builtins.str + @property + def processors(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Processor]: ... + tag: builtins.str + def __init__( + self, + *, + id: builtins.int = ..., + name: builtins.str = ..., + processors: collections.abc.Iterable[global___Processor] | None = ..., + tag: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["id", b"id", "name", b"name", "processors", b"processors", "tag", b"tag"]) -> None: ... + +global___ProcessorBatch = ProcessorBatch + +@typing_extensions.final +class Functor(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class OptionsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + NAME_FIELD_NUMBER: builtins.int + SERDES_FIELD_NUMBER: builtins.int + BODY_FIELD_NUMBER: builtins.int + OPTIONS_FIELD_NUMBER: builtins.int + name: builtins.str + serdes: builtins.str + body: builtins.bytes + @property + def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ... + def __init__( + self, + *, + name: builtins.str = ..., + serdes: builtins.str = ..., + body: builtins.bytes = ..., + options: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["body", b"body", "name", b"name", "options", b"options", "serdes", b"serdes"]) -> None: ... + +global___Functor = Functor + +@typing_extensions.final +class Partitioner(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TYPE_FIELD_NUMBER: builtins.int + BODY_FIELD_NUMBER: builtins.int + type: builtins.int + body: builtins.bytes + def __init__( + self, + *, + type: builtins.int = ..., + body: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["body", b"body", "type", b"type"]) -> None: ... + +global___Partitioner = Partitioner + +@typing_extensions.final +class Serdes(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TYPE_FIELD_NUMBER: builtins.int + BODY_FIELD_NUMBER: builtins.int + type: builtins.int + body: builtins.bytes + def __init__( + self, + *, + type: builtins.int = ..., + body: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["body", b"body", "type", b"type"]) -> None: ... + +global___Serdes = Serdes + +@typing_extensions.final +class Pair(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.bytes + value: builtins.bytes + def __init__( + self, + *, + key: builtins.bytes = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + +global___Pair = Pair + +@typing_extensions.final +class PairBatch(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + PAIRS_FIELD_NUMBER: builtins.int + @property + def pairs(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Pair]: ... + def __init__( + self, + *, + pairs: collections.abc.Iterable[global___Pair] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["pairs", b"pairs"]) -> None: ... + +global___PairBatch = PairBatch + +@typing_extensions.final +class StoreLocator(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ID_FIELD_NUMBER: builtins.int + STORE_TYPE_FIELD_NUMBER: builtins.int + NAMESPACE_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + PATH_FIELD_NUMBER: builtins.int + TOTAL_PARTITIONS_FIELD_NUMBER: builtins.int + KEY_SERDES_TYPE_FIELD_NUMBER: builtins.int + VALUE_SERDES_TYPE_FIELD_NUMBER: builtins.int + PARTITIONER_TYPE_FIELD_NUMBER: builtins.int + id: builtins.int + store_type: builtins.str + namespace: builtins.str + name: builtins.str + path: builtins.str + total_partitions: builtins.int + key_serdes_type: builtins.int + value_serdes_type: builtins.int + partitioner_type: builtins.int + def __init__( + self, + *, + id: builtins.int = ..., + store_type: builtins.str = ..., + namespace: builtins.str = ..., + name: builtins.str = ..., + path: builtins.str = ..., + total_partitions: builtins.int = ..., + key_serdes_type: builtins.int = ..., + value_serdes_type: builtins.int = ..., + partitioner_type: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["id", b"id", "key_serdes_type", b"key_serdes_type", "name", b"name", "namespace", b"namespace", "partitioner_type", b"partitioner_type", "path", b"path", "store_type", b"store_type", "total_partitions", b"total_partitions", "value_serdes_type", b"value_serdes_type"]) -> None: ... + +global___StoreLocator = StoreLocator + +@typing_extensions.final +class Store(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class OptionsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + STORELOCATOR_FIELD_NUMBER: builtins.int + PARTITIONS_FIELD_NUMBER: builtins.int + OPTIONS_FIELD_NUMBER: builtins.int + @property + def storeLocator(self) -> global___StoreLocator: ... + @property + def partitions(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Partition]: ... + @property + def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ... + def __init__( + self, + *, + storeLocator: global___StoreLocator | None = ..., + partitions: collections.abc.Iterable[global___Partition] | None = ..., + options: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["storeLocator", b"storeLocator"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["options", b"options", "partitions", b"partitions", "storeLocator", b"storeLocator"]) -> None: ... + +global___Store = Store + +@typing_extensions.final +class StoreList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STORES_FIELD_NUMBER: builtins.int + @property + def stores(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Store]: ... + def __init__( + self, + *, + stores: collections.abc.Iterable[global___Store] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["stores", b"stores"]) -> None: ... + +global___StoreList = StoreList + +@typing_extensions.final +class Partition(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ID_FIELD_NUMBER: builtins.int + STORELOCATOR_FIELD_NUMBER: builtins.int + PROCESSOR_FIELD_NUMBER: builtins.int + SERVERNODEID_FIELD_NUMBER: builtins.int + RANKINNODE_FIELD_NUMBER: builtins.int + id: builtins.int + @property + def storeLocator(self) -> global___StoreLocator: ... + @property + def processor(self) -> global___Processor: ... + serverNodeId: builtins.int + rankInNode: builtins.int + def __init__( + self, + *, + id: builtins.int = ..., + storeLocator: global___StoreLocator | None = ..., + processor: global___Processor | None = ..., + serverNodeId: builtins.int = ..., + rankInNode: builtins.int = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["processor", b"processor", "storeLocator", b"storeLocator"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["id", b"id", "processor", b"processor", "rankInNode", b"rankInNode", "serverNodeId", b"serverNodeId", "storeLocator", b"storeLocator"]) -> None: ... + +global___Partition = Partition + +@typing_extensions.final +class CallInfo(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CALLSEQ_FIELD_NUMBER: builtins.int + callSeq: builtins.str + def __init__( + self, + *, + callSeq: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["callSeq", b"callSeq"]) -> None: ... + +global___CallInfo = CallInfo + +@typing_extensions.final +class JobIO(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STORE_FIELD_NUMBER: builtins.int + KEY_SERDES_FIELD_NUMBER: builtins.int + VALUE_SERDES_FIELD_NUMBER: builtins.int + PARTITIONER_FIELD_NUMBER: builtins.int + @property + def store(self) -> global___Store: ... + @property + def key_serdes(self) -> global___Serdes: ... + @property + def value_serdes(self) -> global___Serdes: ... + @property + def partitioner(self) -> global___Partitioner: ... + def __init__( + self, + *, + store: global___Store | None = ..., + key_serdes: global___Serdes | None = ..., + value_serdes: global___Serdes | None = ..., + partitioner: global___Partitioner | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["key_serdes", b"key_serdes", "partitioner", b"partitioner", "store", b"store", "value_serdes", b"value_serdes"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["key_serdes", b"key_serdes", "partitioner", b"partitioner", "store", b"store", "value_serdes", b"value_serdes"]) -> None: ... + +global___JobIO = JobIO + +@typing_extensions.final +class Job(google.protobuf.message.Message): + """todo: add / task status""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class OptionsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + ID_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + INPUTS_FIELD_NUMBER: builtins.int + OUTPUTS_FIELD_NUMBER: builtins.int + FUNCTORS_FIELD_NUMBER: builtins.int + OPTIONS_FIELD_NUMBER: builtins.int + id: builtins.str + name: builtins.str + @property + def inputs(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___JobIO]: ... + @property + def outputs(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___JobIO]: ... + @property + def functors(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Functor]: ... + @property + def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ... + def __init__( + self, + *, + id: builtins.str = ..., + name: builtins.str = ..., + inputs: collections.abc.Iterable[global___JobIO] | None = ..., + outputs: collections.abc.Iterable[global___JobIO] | None = ..., + functors: collections.abc.Iterable[global___Functor] | None = ..., + options: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["functors", b"functors", "id", b"id", "inputs", b"inputs", "name", b"name", "options", b"options", "outputs", b"outputs"]) -> None: ... + +global___Job = Job + +@typing_extensions.final +class Task(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ID_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + INPUTS_FIELD_NUMBER: builtins.int + OUTPUTS_FIELD_NUMBER: builtins.int + JOB_FIELD_NUMBER: builtins.int + id: builtins.str + name: builtins.str + @property + def inputs(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Partition]: ... + @property + def outputs(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Partition]: ... + @property + def job(self) -> global___Job: ... + def __init__( + self, + *, + id: builtins.str = ..., + name: builtins.str = ..., + inputs: collections.abc.Iterable[global___Partition] | None = ..., + outputs: collections.abc.Iterable[global___Partition] | None = ..., + job: global___Job | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["job", b"job"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["id", b"id", "inputs", b"inputs", "job", b"job", "name", b"name", "outputs", b"outputs"]) -> None: ... + +global___Task = Task + +@typing_extensions.final +class SessionMeta(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class OptionsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + ID_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + STATUS_FIELD_NUMBER: builtins.int + TAG_FIELD_NUMBER: builtins.int + PROCESSORS_FIELD_NUMBER: builtins.int + OPTIONS_FIELD_NUMBER: builtins.int + id: builtins.str + name: builtins.str + status: builtins.str + tag: builtins.str + @property + def processors(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Processor]: ... + @property + def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ... + def __init__( + self, + *, + id: builtins.str = ..., + name: builtins.str = ..., + status: builtins.str = ..., + tag: builtins.str = ..., + processors: collections.abc.Iterable[global___Processor] | None = ..., + options: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["id", b"id", "name", b"name", "options", b"options", "processors", b"processors", "status", b"status", "tag", b"tag"]) -> None: ... + +global___SessionMeta = SessionMeta + +@typing_extensions.final +class ResourceAllocation(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SERVERNODEID_FIELD_NUMBER: builtins.int + STATUS_FIELD_NUMBER: builtins.int + SESSIONID_FIELD_NUMBER: builtins.int + OPERATETYPE_FIELD_NUMBER: builtins.int + RESOURCES_FIELD_NUMBER: builtins.int + serverNodeId: builtins.int + status: builtins.str + sessionId: builtins.str + operateType: builtins.str + @property + def resources(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Resource]: ... + def __init__( + self, + *, + serverNodeId: builtins.int = ..., + status: builtins.str = ..., + sessionId: builtins.str = ..., + operateType: builtins.str = ..., + resources: collections.abc.Iterable[global___Resource] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["operateType", b"operateType", "resources", b"resources", "serverNodeId", b"serverNodeId", "sessionId", b"sessionId", "status", b"status"]) -> None: ... + +global___ResourceAllocation = ResourceAllocation + +@typing_extensions.final +class Resource(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TYPE_FIELD_NUMBER: builtins.int + TOTAL_FIELD_NUMBER: builtins.int + USED_FIELD_NUMBER: builtins.int + ALLOCATED_FIELD_NUMBER: builtins.int + type: builtins.str + total: builtins.int + used: builtins.int + allocated: builtins.int + def __init__( + self, + *, + type: builtins.str = ..., + total: builtins.int = ..., + used: builtins.int = ..., + allocated: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["allocated", b"allocated", "total", b"total", "type", b"type", "used", b"used"]) -> None: ... + +global___Resource = Resource + +@typing_extensions.final +class NodeHeartbeat(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ID_FIELD_NUMBER: builtins.int + NODE_FIELD_NUMBER: builtins.int + CODE_FIELD_NUMBER: builtins.int + MSG_FIELD_NUMBER: builtins.int + GPUPROCESSORS_FIELD_NUMBER: builtins.int + CPUPROCESSORS_FIELD_NUMBER: builtins.int + id: builtins.int + @property + def node(self) -> global___ServerNode: ... + code: builtins.str + msg: builtins.str + @property + def gpuProcessors(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ... + @property + def cpuProcessors(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ... + def __init__( + self, + *, + id: builtins.int = ..., + node: global___ServerNode | None = ..., + code: builtins.str = ..., + msg: builtins.str = ..., + gpuProcessors: collections.abc.Iterable[builtins.int] | None = ..., + cpuProcessors: collections.abc.Iterable[builtins.int] | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["node", b"node"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["code", b"code", "cpuProcessors", b"cpuProcessors", "gpuProcessors", b"gpuProcessors", "id", b"id", "msg", b"msg", "node", b"node"]) -> None: ... + +global___NodeHeartbeat = NodeHeartbeat + +@typing_extensions.final +class MetaInfoResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class MetaMapEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + METAMAP_FIELD_NUMBER: builtins.int + @property + def metaMap(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ... + def __init__( + self, + *, + metaMap: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["metaMap", b"metaMap"]) -> None: ... + +global___MetaInfoResponse = MetaInfoResponse + +@typing_extensions.final +class MetaInfoRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + key: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key"]) -> None: ... + +global___MetaInfoRequest = MetaInfoRequest + +@typing_extensions.final +class QueueViewResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + QUEUESIZE_FIELD_NUMBER: builtins.int + queueSize: builtins.int + def __init__( + self, + *, + queueSize: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["queueSize", b"queueSize"]) -> None: ... + +global___QueueViewResponse = QueueViewResponse + +@typing_extensions.final +class QueueViewRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + key: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key"]) -> None: ... + +global___QueueViewRequest = QueueViewRequest + +@typing_extensions.final +class CheckResourceEnoughRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RESOURCE_TYPE_FIELD_NUMBER: builtins.int + REQUIRED_RESOURCE_COUNT_FIELD_NUMBER: builtins.int + CHECK_TYPE_FIELD_NUMBER: builtins.int + resource_type: builtins.str + required_resource_count: builtins.int + check_type: builtins.str + def __init__( + self, + *, + resource_type: builtins.str = ..., + required_resource_count: builtins.int = ..., + check_type: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["check_type", b"check_type", "required_resource_count", b"required_resource_count", "resource_type", b"resource_type"]) -> None: ... + +global___CheckResourceEnoughRequest = CheckResourceEnoughRequest + +@typing_extensions.final +class CheckResourceEnoughResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + IS_ENOUGH_FIELD_NUMBER: builtins.int + CLUSTERINFO_FIELD_NUMBER: builtins.int + is_enough: builtins.bool + @property + def clusterInfo(self) -> global___ServerCluster: ... + def __init__( + self, + *, + is_enough: builtins.bool = ..., + clusterInfo: global___ServerCluster | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["clusterInfo", b"clusterInfo"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["clusterInfo", b"clusterInfo", "is_enough", b"is_enough"]) -> None: ... + +global___CheckResourceEnoughResponse = CheckResourceEnoughResponse + +@typing_extensions.final +class queryClusterInfoRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___queryClusterInfoRequest = queryClusterInfoRequest + +@typing_extensions.final +class queryClusterInfoResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CLUSTERINFO_FIELD_NUMBER: builtins.int + @property + def clusterInfo(self) -> global___ServerCluster: ... + def __init__( + self, + *, + clusterInfo: global___ServerCluster | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["clusterInfo", b"clusterInfo"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["clusterInfo", b"clusterInfo"]) -> None: ... + +global___queryClusterInfoResponse = queryClusterInfoResponse diff --git a/python/eggroll/backport/proto/proxy_pb2.py b/python/eggroll/backport/proto/proxy_pb2.py new file mode 100644 index 000000000..4c8b785d7 --- /dev/null +++ b/python/eggroll/backport/proto/proxy_pb2.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: proxy.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +import basic_meta_pb2 as basic__meta__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0bproxy.proto\x12*com.webank.ai.eggroll.api.networking.proxy\x1a\x10\x62\x61sic-meta.proto\"&\n\x05Model\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07\x64\x61taKey\x18\x02 \x01(\t\"X\n\x04Task\x12\x0e\n\x06taskId\x18\x01 \x01(\t\x12@\n\x05model\x18\x02 \x01(\x0b\x32\x31.com.webank.ai.eggroll.api.networking.proxy.Model\"p\n\x05Topic\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07partyId\x18\x02 \x01(\t\x12\x0c\n\x04role\x18\x03 \x01(\t\x12:\n\x08\x63\x61llback\x18\x04 \x01(\x0b\x32(.com.webank.ai.eggroll.api.core.Endpoint\"\x17\n\x07\x43ommand\x12\x0c\n\x04name\x18\x01 \x01(\t\"p\n\x04\x43onf\x12\x16\n\x0eoverallTimeout\x18\x01 \x01(\x03\x12\x1d\n\x15\x63ompletionWaitTimeout\x18\x02 \x01(\x03\x12\x1d\n\x15packetIntervalTimeout\x18\x03 \x01(\x03\x12\x12\n\nmaxRetries\x18\x04 \x01(\x05\"\x9a\x03\n\x08Metadata\x12>\n\x04task\x18\x01 \x01(\x0b\x32\x30.com.webank.ai.eggroll.api.networking.proxy.Task\x12>\n\x03src\x18\x02 \x01(\x0b\x32\x31.com.webank.ai.eggroll.api.networking.proxy.Topic\x12>\n\x03\x64st\x18\x03 \x01(\x0b\x32\x31.com.webank.ai.eggroll.api.networking.proxy.Topic\x12\x44\n\x07\x63ommand\x18\x04 \x01(\x0b\x32\x33.com.webank.ai.eggroll.api.networking.proxy.Command\x12\x10\n\x08operator\x18\x05 \x01(\t\x12\x0b\n\x03seq\x18\x06 \x01(\x03\x12\x0b\n\x03\x61\x63k\x18\x07 \x01(\x03\x12>\n\x04\x63onf\x18\x08 \x01(\x0b\x32\x30.com.webank.ai.eggroll.api.networking.proxy.Conf\x12\x0b\n\x03\x65xt\x18\t \x01(\x0c\x12\x0f\n\x07version\x18\x64 \x01(\t\"\"\n\x04\x44\x61ta\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c\"\x8e\x01\n\x06Packet\x12\x44\n\x06header\x18\x01 \x01(\x0b\x32\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata\x12>\n\x04\x62ody\x18\x02 \x01(\x0b\x32\x30.com.webank.ai.eggroll.api.networking.proxy.Data\"\xa3\x01\n\x11HeartbeatResponse\x12\x44\n\x06header\x18\x01 \x01(\x0b\x32\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata\x12H\n\toperation\x18\x02 \x01(\x0e\x32\x35.com.webank.ai.eggroll.api.networking.proxy.Operation\"\xc5\x01\n\x0cPollingFrame\x12\x0e\n\x06method\x18\x01 \x01(\t\x12\x0b\n\x03seq\x18\x02 \x01(\x03\x12\x46\n\x08metadata\x18\n \x01(\x0b\x32\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata\x12\x42\n\x06packet\x18\x14 \x01(\x0b\x32\x32.com.webank.ai.eggroll.api.networking.proxy.Packet\x12\x0c\n\x04\x64\x65sc\x18\x1e \x01(\t*O\n\tOperation\x12\t\n\x05START\x10\x00\x12\x07\n\x03RUN\x10\x01\x12\x08\n\x04STOP\x10\x02\x12\x08\n\x04KILL\x10\x03\x12\x0c\n\x08GET_DATA\x10\x04\x12\x0c\n\x08PUT_DATA\x10\x05\x32\xf6\x03\n\x13\x44\x61taTransferService\x12r\n\x04push\x12\x32.com.webank.ai.eggroll.api.networking.proxy.Packet\x1a\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata(\x01\x12r\n\x04pull\x12\x34.com.webank.ai.eggroll.api.networking.proxy.Metadata\x1a\x32.com.webank.ai.eggroll.api.networking.proxy.Packet0\x01\x12s\n\tunaryCall\x12\x32.com.webank.ai.eggroll.api.networking.proxy.Packet\x1a\x32.com.webank.ai.eggroll.api.networking.proxy.Packet\x12\x81\x01\n\x07polling\x12\x38.com.webank.ai.eggroll.api.networking.proxy.PollingFrame\x1a\x38.com.webank.ai.eggroll.api.networking.proxy.PollingFrame(\x01\x30\x01\x32t\n\x0cRouteService\x12\x64\n\x05query\x12\x31.com.webank.ai.eggroll.api.networking.proxy.Topic\x1a(.com.webank.ai.eggroll.api.core.Endpointb\x06proto3') + +_OPERATION = DESCRIPTOR.enum_types_by_name['Operation'] +Operation = enum_type_wrapper.EnumTypeWrapper(_OPERATION) +START = 0 +RUN = 1 +STOP = 2 +KILL = 3 +GET_DATA = 4 +PUT_DATA = 5 + + +_MODEL = DESCRIPTOR.message_types_by_name['Model'] +_TASK = DESCRIPTOR.message_types_by_name['Task'] +_TOPIC = DESCRIPTOR.message_types_by_name['Topic'] +_COMMAND = DESCRIPTOR.message_types_by_name['Command'] +_CONF = DESCRIPTOR.message_types_by_name['Conf'] +_METADATA = DESCRIPTOR.message_types_by_name['Metadata'] +_DATA = DESCRIPTOR.message_types_by_name['Data'] +_PACKET = DESCRIPTOR.message_types_by_name['Packet'] +_HEARTBEATRESPONSE = DESCRIPTOR.message_types_by_name['HeartbeatResponse'] +_POLLINGFRAME = DESCRIPTOR.message_types_by_name['PollingFrame'] +Model = _reflection.GeneratedProtocolMessageType('Model', (_message.Message,), { + 'DESCRIPTOR' : _MODEL, + '__module__' : 'proxy_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Model) + }) +_sym_db.RegisterMessage(Model) + +Task = _reflection.GeneratedProtocolMessageType('Task', (_message.Message,), { + 'DESCRIPTOR' : _TASK, + '__module__' : 'proxy_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Task) + }) +_sym_db.RegisterMessage(Task) + +Topic = _reflection.GeneratedProtocolMessageType('Topic', (_message.Message,), { + 'DESCRIPTOR' : _TOPIC, + '__module__' : 'proxy_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Topic) + }) +_sym_db.RegisterMessage(Topic) + +Command = _reflection.GeneratedProtocolMessageType('Command', (_message.Message,), { + 'DESCRIPTOR' : _COMMAND, + '__module__' : 'proxy_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Command) + }) +_sym_db.RegisterMessage(Command) + +Conf = _reflection.GeneratedProtocolMessageType('Conf', (_message.Message,), { + 'DESCRIPTOR' : _CONF, + '__module__' : 'proxy_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Conf) + }) +_sym_db.RegisterMessage(Conf) + +Metadata = _reflection.GeneratedProtocolMessageType('Metadata', (_message.Message,), { + 'DESCRIPTOR' : _METADATA, + '__module__' : 'proxy_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Metadata) + }) +_sym_db.RegisterMessage(Metadata) + +Data = _reflection.GeneratedProtocolMessageType('Data', (_message.Message,), { + 'DESCRIPTOR' : _DATA, + '__module__' : 'proxy_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Data) + }) +_sym_db.RegisterMessage(Data) + +Packet = _reflection.GeneratedProtocolMessageType('Packet', (_message.Message,), { + 'DESCRIPTOR' : _PACKET, + '__module__' : 'proxy_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.Packet) + }) +_sym_db.RegisterMessage(Packet) + +HeartbeatResponse = _reflection.GeneratedProtocolMessageType('HeartbeatResponse', (_message.Message,), { + 'DESCRIPTOR' : _HEARTBEATRESPONSE, + '__module__' : 'proxy_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.HeartbeatResponse) + }) +_sym_db.RegisterMessage(HeartbeatResponse) + +PollingFrame = _reflection.GeneratedProtocolMessageType('PollingFrame', (_message.Message,), { + 'DESCRIPTOR' : _POLLINGFRAME, + '__module__' : 'proxy_pb2' + # @@protoc_insertion_point(class_scope:com.webank.ai.eggroll.api.networking.proxy.PollingFrame) + }) +_sym_db.RegisterMessage(PollingFrame) + +_DATATRANSFERSERVICE = DESCRIPTOR.services_by_name['DataTransferService'] +_ROUTESERVICE = DESCRIPTOR.services_by_name['RouteService'] +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _OPERATION._serialized_start=1420 + _OPERATION._serialized_end=1499 + _MODEL._serialized_start=77 + _MODEL._serialized_end=115 + _TASK._serialized_start=117 + _TASK._serialized_end=205 + _TOPIC._serialized_start=207 + _TOPIC._serialized_end=319 + _COMMAND._serialized_start=321 + _COMMAND._serialized_end=344 + _CONF._serialized_start=346 + _CONF._serialized_end=458 + _METADATA._serialized_start=461 + _METADATA._serialized_end=871 + _DATA._serialized_start=873 + _DATA._serialized_end=907 + _PACKET._serialized_start=910 + _PACKET._serialized_end=1052 + _HEARTBEATRESPONSE._serialized_start=1055 + _HEARTBEATRESPONSE._serialized_end=1218 + _POLLINGFRAME._serialized_start=1221 + _POLLINGFRAME._serialized_end=1418 + _DATATRANSFERSERVICE._serialized_start=1502 + _DATATRANSFERSERVICE._serialized_end=2004 + _ROUTESERVICE._serialized_start=2006 + _ROUTESERVICE._serialized_end=2122 +# @@protoc_insertion_point(module_scope) diff --git a/python/eggroll/backport/proto/proxy_pb2.pyi b/python/eggroll/backport/proto/proxy_pb2.pyi new file mode 100644 index 000000000..99f44e874 --- /dev/null +++ b/python/eggroll/backport/proto/proxy_pb2.pyi @@ -0,0 +1,326 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file + +Copyright 2019 The Eggroll Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import basic_meta_pb2 +import builtins +import google.protobuf.descriptor +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message +import sys +import typing + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _Operation: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _OperationEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_Operation.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + START: _Operation.ValueType # 0 + RUN: _Operation.ValueType # 1 + STOP: _Operation.ValueType # 2 + KILL: _Operation.ValueType # 3 + GET_DATA: _Operation.ValueType # 4 + PUT_DATA: _Operation.ValueType # 5 + +class Operation(_Operation, metaclass=_OperationEnumTypeWrapper): + """returned by service heartbeat to decide next operation""" + +START: Operation.ValueType # 0 +RUN: Operation.ValueType # 1 +STOP: Operation.ValueType # 2 +KILL: Operation.ValueType # 3 +GET_DATA: Operation.ValueType # 4 +PUT_DATA: Operation.ValueType # 5 +global___Operation = Operation + +@typing_extensions.final +class Model(google.protobuf.message.Message): + """metadata of event""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAME_FIELD_NUMBER: builtins.int + DATAKEY_FIELD_NUMBER: builtins.int + name: builtins.str + dataKey: builtins.str + def __init__( + self, + *, + name: builtins.str = ..., + dataKey: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["dataKey", b"dataKey", "name", b"name"]) -> None: ... + +global___Model = Model + +@typing_extensions.final +class Task(google.protobuf.message.Message): + """metadata of task""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TASKID_FIELD_NUMBER: builtins.int + MODEL_FIELD_NUMBER: builtins.int + taskId: builtins.str + @property + def model(self) -> global___Model: ... + def __init__( + self, + *, + taskId: builtins.str = ..., + model: global___Model | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["model", b"model"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["model", b"model", "taskId", b"taskId"]) -> None: ... + +global___Task = Task + +@typing_extensions.final +class Topic(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAME_FIELD_NUMBER: builtins.int + PARTYID_FIELD_NUMBER: builtins.int + ROLE_FIELD_NUMBER: builtins.int + CALLBACK_FIELD_NUMBER: builtins.int + name: builtins.str + partyId: builtins.str + role: builtins.str + @property + def callback(self) -> basic_meta_pb2.Endpoint: + """implication of pub/sub model, necessary for http-based senario""" + def __init__( + self, + *, + name: builtins.str = ..., + partyId: builtins.str = ..., + role: builtins.str = ..., + callback: basic_meta_pb2.Endpoint | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["callback", b"callback"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["callback", b"callback", "name", b"name", "partyId", b"partyId", "role", b"role"]) -> None: ... + +global___Topic = Topic + +@typing_extensions.final +class Command(google.protobuf.message.Message): + """task admin command""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NAME_FIELD_NUMBER: builtins.int + name: builtins.str + def __init__( + self, + *, + name: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["name", b"name"]) -> None: ... + +global___Command = Command + +@typing_extensions.final +class Conf(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + OVERALLTIMEOUT_FIELD_NUMBER: builtins.int + COMPLETIONWAITTIMEOUT_FIELD_NUMBER: builtins.int + PACKETINTERVALTIMEOUT_FIELD_NUMBER: builtins.int + MAXRETRIES_FIELD_NUMBER: builtins.int + overallTimeout: builtins.int + """total timeout, in ms""" + completionWaitTimeout: builtins.int + """timeout for waiting for complete, in ms""" + packetIntervalTimeout: builtins.int + """timeout for packet interval, in ms""" + maxRetries: builtins.int + def __init__( + self, + *, + overallTimeout: builtins.int = ..., + completionWaitTimeout: builtins.int = ..., + packetIntervalTimeout: builtins.int = ..., + maxRetries: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["completionWaitTimeout", b"completionWaitTimeout", "maxRetries", b"maxRetries", "overallTimeout", b"overallTimeout", "packetIntervalTimeout", b"packetIntervalTimeout"]) -> None: ... + +global___Conf = Conf + +@typing_extensions.final +class Metadata(google.protobuf.message.Message): + """metadata used for network data transfer""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TASK_FIELD_NUMBER: builtins.int + SRC_FIELD_NUMBER: builtins.int + DST_FIELD_NUMBER: builtins.int + COMMAND_FIELD_NUMBER: builtins.int + OPERATOR_FIELD_NUMBER: builtins.int + SEQ_FIELD_NUMBER: builtins.int + ACK_FIELD_NUMBER: builtins.int + CONF_FIELD_NUMBER: builtins.int + EXT_FIELD_NUMBER: builtins.int + VERSION_FIELD_NUMBER: builtins.int + @property + def task(self) -> global___Task: + """task description""" + @property + def src(self) -> global___Topic: + """source topic""" + @property + def dst(self) -> global___Topic: + """destincation topic""" + @property + def command(self) -> global___Command: + """task managing command (if any)""" + operator: builtins.str + """model operator""" + seq: builtins.int + """stream seq (reserved)""" + ack: builtins.int + """stream ack (reserved)""" + @property + def conf(self) -> global___Conf: + """operation config""" + ext: builtins.bytes + version: builtins.str + def __init__( + self, + *, + task: global___Task | None = ..., + src: global___Topic | None = ..., + dst: global___Topic | None = ..., + command: global___Command | None = ..., + operator: builtins.str = ..., + seq: builtins.int = ..., + ack: builtins.int = ..., + conf: global___Conf | None = ..., + ext: builtins.bytes = ..., + version: builtins.str = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["command", b"command", "conf", b"conf", "dst", b"dst", "src", b"src", "task", b"task"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["ack", b"ack", "command", b"command", "conf", b"conf", "dst", b"dst", "ext", b"ext", "operator", b"operator", "seq", b"seq", "src", b"src", "task", b"task", "version", b"version"]) -> None: ... + +global___Metadata = Metadata + +@typing_extensions.final +class Data(google.protobuf.message.Message): + """includes key and value field, supporting sequential and random data transfer""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + """compatible with list / dict""" + value: builtins.bytes + """actual value""" + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + +global___Data = Data + +@typing_extensions.final +class Packet(google.protobuf.message.Message): + """data streaming packet""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + HEADER_FIELD_NUMBER: builtins.int + BODY_FIELD_NUMBER: builtins.int + @property + def header(self) -> global___Metadata: + """packet header""" + @property + def body(self) -> global___Data: + """packet body""" + def __init__( + self, + *, + header: global___Metadata | None = ..., + body: global___Data | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["body", b"body", "header", b"header"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["body", b"body", "header", b"header"]) -> None: ... + +global___Packet = Packet + +@typing_extensions.final +class HeartbeatResponse(google.protobuf.message.Message): + """response of heartbeat""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + HEADER_FIELD_NUMBER: builtins.int + OPERATION_FIELD_NUMBER: builtins.int + @property + def header(self) -> global___Metadata: ... + operation: global___Operation.ValueType + def __init__( + self, + *, + header: global___Metadata | None = ..., + operation: global___Operation.ValueType = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["header", b"header"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["header", b"header", "operation", b"operation"]) -> None: ... + +global___HeartbeatResponse = HeartbeatResponse + +@typing_extensions.final +class PollingFrame(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + METHOD_FIELD_NUMBER: builtins.int + SEQ_FIELD_NUMBER: builtins.int + METADATA_FIELD_NUMBER: builtins.int + PACKET_FIELD_NUMBER: builtins.int + DESC_FIELD_NUMBER: builtins.int + method: builtins.str + seq: builtins.int + @property + def metadata(self) -> global___Metadata: ... + @property + def packet(self) -> global___Packet: ... + desc: builtins.str + def __init__( + self, + *, + method: builtins.str = ..., + seq: builtins.int = ..., + metadata: global___Metadata | None = ..., + packet: global___Packet | None = ..., + desc: builtins.str = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["metadata", b"metadata", "packet", b"packet"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["desc", b"desc", "metadata", b"metadata", "method", b"method", "packet", b"packet", "seq", b"seq"]) -> None: ... + +global___PollingFrame = PollingFrame diff --git a/python/eggroll/backport/proto/proxy_pb2_grpc.py b/python/eggroll/backport/proto/proxy_pb2_grpc.py new file mode 100644 index 000000000..d9f99065c --- /dev/null +++ b/python/eggroll/backport/proto/proxy_pb2_grpc.py @@ -0,0 +1,230 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +import basic_meta_pb2 as basic__meta__pb2 +import proxy_pb2 as proxy__pb2 + + +class DataTransferServiceStub(object): + """data transfer service + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.push = channel.stream_unary( + '/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/push', + request_serializer=proxy__pb2.Packet.SerializeToString, + response_deserializer=proxy__pb2.Metadata.FromString, + ) + self.pull = channel.unary_stream( + '/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/pull', + request_serializer=proxy__pb2.Metadata.SerializeToString, + response_deserializer=proxy__pb2.Packet.FromString, + ) + self.unaryCall = channel.unary_unary( + '/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/unaryCall', + request_serializer=proxy__pb2.Packet.SerializeToString, + response_deserializer=proxy__pb2.Packet.FromString, + ) + self.polling = channel.stream_stream( + '/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/polling', + request_serializer=proxy__pb2.PollingFrame.SerializeToString, + response_deserializer=proxy__pb2.PollingFrame.FromString, + ) + + +class DataTransferServiceServicer(object): + """data transfer service + """ + + def push(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def pull(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def unaryCall(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def polling(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_DataTransferServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'push': grpc.stream_unary_rpc_method_handler( + servicer.push, + request_deserializer=proxy__pb2.Packet.FromString, + response_serializer=proxy__pb2.Metadata.SerializeToString, + ), + 'pull': grpc.unary_stream_rpc_method_handler( + servicer.pull, + request_deserializer=proxy__pb2.Metadata.FromString, + response_serializer=proxy__pb2.Packet.SerializeToString, + ), + 'unaryCall': grpc.unary_unary_rpc_method_handler( + servicer.unaryCall, + request_deserializer=proxy__pb2.Packet.FromString, + response_serializer=proxy__pb2.Packet.SerializeToString, + ), + 'polling': grpc.stream_stream_rpc_method_handler( + servicer.polling, + request_deserializer=proxy__pb2.PollingFrame.FromString, + response_serializer=proxy__pb2.PollingFrame.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'com.webank.ai.eggroll.api.networking.proxy.DataTransferService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class DataTransferService(object): + """data transfer service + """ + + @staticmethod + def push(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_unary(request_iterator, target, '/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/push', + proxy__pb2.Packet.SerializeToString, + proxy__pb2.Metadata.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def pull(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/pull', + proxy__pb2.Metadata.SerializeToString, + proxy__pb2.Packet.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def unaryCall(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/unaryCall', + proxy__pb2.Packet.SerializeToString, + proxy__pb2.Packet.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def polling(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_stream(request_iterator, target, '/com.webank.ai.eggroll.api.networking.proxy.DataTransferService/polling', + proxy__pb2.PollingFrame.SerializeToString, + proxy__pb2.PollingFrame.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + +class RouteServiceStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.query = channel.unary_unary( + '/com.webank.ai.eggroll.api.networking.proxy.RouteService/query', + request_serializer=proxy__pb2.Topic.SerializeToString, + response_deserializer=basic__meta__pb2.Endpoint.FromString, + ) + + +class RouteServiceServicer(object): + """Missing associated documentation comment in .proto file.""" + + def query(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_RouteServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'query': grpc.unary_unary_rpc_method_handler( + servicer.query, + request_deserializer=proxy__pb2.Topic.FromString, + response_serializer=basic__meta__pb2.Endpoint.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'com.webank.ai.eggroll.api.networking.proxy.RouteService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class RouteService(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def query(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/com.webank.ai.eggroll.api.networking.proxy.RouteService/query', + proxy__pb2.Topic.SerializeToString, + basic__meta__pb2.Endpoint.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/python/eggroll/backport/proto/transfer_pb2.py b/python/eggroll/backport/proto/transfer_pb2.py new file mode 100644 index 000000000..822cced8a --- /dev/null +++ b/python/eggroll/backport/proto/transfer_pb2.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: transfer.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0etransfer.proto\x12 com.webank.eggroll.core.transfer\"Y\n\x0eTransferHeader\x12\n\n\x02id\x18\x01 \x01(\x05\x12\x0b\n\x03tag\x18\x02 \x01(\t\x12\x11\n\ttotalSize\x18\x03 \x01(\x03\x12\x0e\n\x06status\x18\x04 \x01(\t\x12\x0b\n\x03\x65xt\x18\x05 \x01(\x0c\"r\n\rTransferBatch\x12@\n\x06header\x18\x01 \x01(\x0b\x32\x30.com.webank.eggroll.core.transfer.TransferHeader\x12\x11\n\tbatchSize\x18\x02 \x01(\x03\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\"\xb0\x03\n\x0eRollSiteHeader\x12\x19\n\x11rollSiteSessionId\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0b\n\x03tag\x18\x03 \x01(\t\x12\x0f\n\x07srcRole\x18\x04 \x01(\t\x12\x12\n\nsrcPartyId\x18\x05 \x01(\t\x12\x0f\n\x07\x64stRole\x18\x06 \x01(\t\x12\x12\n\ndstPartyId\x18\x07 \x01(\t\x12\x10\n\x08\x64\x61taType\x18\x08 \x01(\t\x12N\n\x07options\x18\n \x03(\x0b\x32=.com.webank.eggroll.core.transfer.RollSiteHeader.OptionsEntry\x12\x17\n\x0ftotalPartitions\x18\x0f \x01(\x05\x12\x13\n\x0bpartitionId\x18\x10 \x01(\x05\x12\x14\n\x0ctotalStreams\x18\x11 \x01(\x03\x12\x14\n\x0ctotalBatches\x18\x12 \x01(\x03\x12\x11\n\tstreamSeq\x18\x14 \x01(\x03\x12\x10\n\x08\x62\x61tchSeq\x18\x15 \x01(\x03\x12\r\n\x05stage\x18\x1e \x01(\t\x1a.\n\x0cOptionsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"<\n\x1cRollSitePullGetHeaderRequest\x12\x0b\n\x03tag\x18\x01 \x01(\t\x12\x0f\n\x07timeout\x18\x02 \x01(\x01\"a\n\x1dRollSitePullGetHeaderResponse\x12@\n\x06header\x18\x01 \x01(\x0b\x32\x30.com.webank.eggroll.core.transfer.RollSiteHeader\"E\n%RollSitePullGetPartitionStatusRequest\x12\x0b\n\x03tag\x18\x01 \x01(\t\x12\x0f\n\x07timeout\x18\x02 \x01(\x01\"\xa0\x06\n&RollSitePullGetPartitionStatusResponse\x12\x14\n\x0cpartition_id\x18\x01 \x01(\x03\x12\x85\x01\n\x06status\x18\x02 \x01(\x0b\x32u.com.webank.eggroll.core.transfer.RollSitePullGetPartitionStatusResponse.RollSitePullGetPartitionStatusResponseStatus\x1a\x30\n\x12IntKeyIntValuePair\x12\x0b\n\x03key\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\x03\x1a\xa5\x04\n,RollSitePullGetPartitionStatusResponseStatus\x12\x0b\n\x03tag\x18\x01 \x01(\t\x12\x13\n\x0bis_finished\x18\x02 \x01(\x08\x12\x15\n\rtotal_batches\x18\x03 \x01(\x03\x12~\n\x19\x62\x61tch_seq_to_pair_counter\x18\x04 \x03(\x0b\x32[.com.webank.eggroll.core.transfer.RollSitePullGetPartitionStatusResponse.IntKeyIntValuePair\x12\x15\n\rtotal_streams\x18\x05 \x01(\x03\x12\x7f\n\x1astream_seq_to_pair_counter\x18\x06 \x03(\x0b\x32[.com.webank.eggroll.core.transfer.RollSitePullGetPartitionStatusResponse.IntKeyIntValuePair\x12|\n\x17stream_seq_to_batch_seq\x18\x07 \x03(\x0b\x32[.com.webank.eggroll.core.transfer.RollSitePullGetPartitionStatusResponse.IntKeyIntValuePair\x12\x13\n\x0btotal_pairs\x18\x08 \x01(\x03\x12\x11\n\tdata_type\x18\t \x01(\t\"-\n\x1eRollSitePullClearStatusRequest\x12\x0b\n\x03tag\x18\x01 \x01(\t\"!\n\x1fRollSitePullClearStatusResponse2\xdb\x02\n\x0fTransferService\x12j\n\x04send\x12/.com.webank.eggroll.core.transfer.TransferBatch\x1a/.com.webank.eggroll.core.transfer.TransferBatch(\x01\x12j\n\x04recv\x12/.com.webank.eggroll.core.transfer.TransferBatch\x1a/.com.webank.eggroll.core.transfer.TransferBatch0\x01\x12p\n\x08sendRecv\x12/.com.webank.eggroll.core.transfer.TransferBatch\x1a/.com.webank.eggroll.core.transfer.TransferBatch(\x01\x30\x01\x62\x06proto3') + + + +_TRANSFERHEADER = DESCRIPTOR.message_types_by_name['TransferHeader'] +_TRANSFERBATCH = DESCRIPTOR.message_types_by_name['TransferBatch'] +_ROLLSITEHEADER = DESCRIPTOR.message_types_by_name['RollSiteHeader'] +_ROLLSITEHEADER_OPTIONSENTRY = _ROLLSITEHEADER.nested_types_by_name['OptionsEntry'] +_ROLLSITEPULLGETHEADERREQUEST = DESCRIPTOR.message_types_by_name['RollSitePullGetHeaderRequest'] +_ROLLSITEPULLGETHEADERRESPONSE = DESCRIPTOR.message_types_by_name['RollSitePullGetHeaderResponse'] +_ROLLSITEPULLGETPARTITIONSTATUSREQUEST = DESCRIPTOR.message_types_by_name['RollSitePullGetPartitionStatusRequest'] +_ROLLSITEPULLGETPARTITIONSTATUSRESPONSE = DESCRIPTOR.message_types_by_name['RollSitePullGetPartitionStatusResponse'] +_ROLLSITEPULLGETPARTITIONSTATUSRESPONSE_INTKEYINTVALUEPAIR = _ROLLSITEPULLGETPARTITIONSTATUSRESPONSE.nested_types_by_name['IntKeyIntValuePair'] +_ROLLSITEPULLGETPARTITIONSTATUSRESPONSE_ROLLSITEPULLGETPARTITIONSTATUSRESPONSESTATUS = _ROLLSITEPULLGETPARTITIONSTATUSRESPONSE.nested_types_by_name['RollSitePullGetPartitionStatusResponseStatus'] +_ROLLSITEPULLCLEARSTATUSREQUEST = DESCRIPTOR.message_types_by_name['RollSitePullClearStatusRequest'] +_ROLLSITEPULLCLEARSTATUSRESPONSE = DESCRIPTOR.message_types_by_name['RollSitePullClearStatusResponse'] +TransferHeader = _reflection.GeneratedProtocolMessageType('TransferHeader', (_message.Message,), { + 'DESCRIPTOR' : _TRANSFERHEADER, + '__module__' : 'transfer_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.transfer.TransferHeader) + }) +_sym_db.RegisterMessage(TransferHeader) + +TransferBatch = _reflection.GeneratedProtocolMessageType('TransferBatch', (_message.Message,), { + 'DESCRIPTOR' : _TRANSFERBATCH, + '__module__' : 'transfer_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.transfer.TransferBatch) + }) +_sym_db.RegisterMessage(TransferBatch) + +RollSiteHeader = _reflection.GeneratedProtocolMessageType('RollSiteHeader', (_message.Message,), { + + 'OptionsEntry' : _reflection.GeneratedProtocolMessageType('OptionsEntry', (_message.Message,), { + 'DESCRIPTOR' : _ROLLSITEHEADER_OPTIONSENTRY, + '__module__' : 'transfer_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.transfer.RollSiteHeader.OptionsEntry) + }) + , + 'DESCRIPTOR' : _ROLLSITEHEADER, + '__module__' : 'transfer_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.transfer.RollSiteHeader) + }) +_sym_db.RegisterMessage(RollSiteHeader) +_sym_db.RegisterMessage(RollSiteHeader.OptionsEntry) + +RollSitePullGetHeaderRequest = _reflection.GeneratedProtocolMessageType('RollSitePullGetHeaderRequest', (_message.Message,), { + 'DESCRIPTOR' : _ROLLSITEPULLGETHEADERREQUEST, + '__module__' : 'transfer_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.transfer.RollSitePullGetHeaderRequest) + }) +_sym_db.RegisterMessage(RollSitePullGetHeaderRequest) + +RollSitePullGetHeaderResponse = _reflection.GeneratedProtocolMessageType('RollSitePullGetHeaderResponse', (_message.Message,), { + 'DESCRIPTOR' : _ROLLSITEPULLGETHEADERRESPONSE, + '__module__' : 'transfer_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.transfer.RollSitePullGetHeaderResponse) + }) +_sym_db.RegisterMessage(RollSitePullGetHeaderResponse) + +RollSitePullGetPartitionStatusRequest = _reflection.GeneratedProtocolMessageType('RollSitePullGetPartitionStatusRequest', (_message.Message,), { + 'DESCRIPTOR' : _ROLLSITEPULLGETPARTITIONSTATUSREQUEST, + '__module__' : 'transfer_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.transfer.RollSitePullGetPartitionStatusRequest) + }) +_sym_db.RegisterMessage(RollSitePullGetPartitionStatusRequest) + +RollSitePullGetPartitionStatusResponse = _reflection.GeneratedProtocolMessageType('RollSitePullGetPartitionStatusResponse', (_message.Message,), { + + 'IntKeyIntValuePair' : _reflection.GeneratedProtocolMessageType('IntKeyIntValuePair', (_message.Message,), { + 'DESCRIPTOR' : _ROLLSITEPULLGETPARTITIONSTATUSRESPONSE_INTKEYINTVALUEPAIR, + '__module__' : 'transfer_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.transfer.RollSitePullGetPartitionStatusResponse.IntKeyIntValuePair) + }) + , + + 'RollSitePullGetPartitionStatusResponseStatus' : _reflection.GeneratedProtocolMessageType('RollSitePullGetPartitionStatusResponseStatus', (_message.Message,), { + 'DESCRIPTOR' : _ROLLSITEPULLGETPARTITIONSTATUSRESPONSE_ROLLSITEPULLGETPARTITIONSTATUSRESPONSESTATUS, + '__module__' : 'transfer_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.transfer.RollSitePullGetPartitionStatusResponse.RollSitePullGetPartitionStatusResponseStatus) + }) + , + 'DESCRIPTOR' : _ROLLSITEPULLGETPARTITIONSTATUSRESPONSE, + '__module__' : 'transfer_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.transfer.RollSitePullGetPartitionStatusResponse) + }) +_sym_db.RegisterMessage(RollSitePullGetPartitionStatusResponse) +_sym_db.RegisterMessage(RollSitePullGetPartitionStatusResponse.IntKeyIntValuePair) +_sym_db.RegisterMessage(RollSitePullGetPartitionStatusResponse.RollSitePullGetPartitionStatusResponseStatus) + +RollSitePullClearStatusRequest = _reflection.GeneratedProtocolMessageType('RollSitePullClearStatusRequest', (_message.Message,), { + 'DESCRIPTOR' : _ROLLSITEPULLCLEARSTATUSREQUEST, + '__module__' : 'transfer_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.transfer.RollSitePullClearStatusRequest) + }) +_sym_db.RegisterMessage(RollSitePullClearStatusRequest) + +RollSitePullClearStatusResponse = _reflection.GeneratedProtocolMessageType('RollSitePullClearStatusResponse', (_message.Message,), { + 'DESCRIPTOR' : _ROLLSITEPULLCLEARSTATUSRESPONSE, + '__module__' : 'transfer_pb2' + # @@protoc_insertion_point(class_scope:com.webank.eggroll.core.transfer.RollSitePullClearStatusResponse) + }) +_sym_db.RegisterMessage(RollSitePullClearStatusResponse) + +_TRANSFERSERVICE = DESCRIPTOR.services_by_name['TransferService'] +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _ROLLSITEHEADER_OPTIONSENTRY._options = None + _ROLLSITEHEADER_OPTIONSENTRY._serialized_options = b'8\001' + _TRANSFERHEADER._serialized_start=52 + _TRANSFERHEADER._serialized_end=141 + _TRANSFERBATCH._serialized_start=143 + _TRANSFERBATCH._serialized_end=257 + _ROLLSITEHEADER._serialized_start=260 + _ROLLSITEHEADER._serialized_end=692 + _ROLLSITEHEADER_OPTIONSENTRY._serialized_start=646 + _ROLLSITEHEADER_OPTIONSENTRY._serialized_end=692 + _ROLLSITEPULLGETHEADERREQUEST._serialized_start=694 + _ROLLSITEPULLGETHEADERREQUEST._serialized_end=754 + _ROLLSITEPULLGETHEADERRESPONSE._serialized_start=756 + _ROLLSITEPULLGETHEADERRESPONSE._serialized_end=853 + _ROLLSITEPULLGETPARTITIONSTATUSREQUEST._serialized_start=855 + _ROLLSITEPULLGETPARTITIONSTATUSREQUEST._serialized_end=924 + _ROLLSITEPULLGETPARTITIONSTATUSRESPONSE._serialized_start=927 + _ROLLSITEPULLGETPARTITIONSTATUSRESPONSE._serialized_end=1727 + _ROLLSITEPULLGETPARTITIONSTATUSRESPONSE_INTKEYINTVALUEPAIR._serialized_start=1127 + _ROLLSITEPULLGETPARTITIONSTATUSRESPONSE_INTKEYINTVALUEPAIR._serialized_end=1175 + _ROLLSITEPULLGETPARTITIONSTATUSRESPONSE_ROLLSITEPULLGETPARTITIONSTATUSRESPONSESTATUS._serialized_start=1178 + _ROLLSITEPULLGETPARTITIONSTATUSRESPONSE_ROLLSITEPULLGETPARTITIONSTATUSRESPONSESTATUS._serialized_end=1727 + _ROLLSITEPULLCLEARSTATUSREQUEST._serialized_start=1729 + _ROLLSITEPULLCLEARSTATUSREQUEST._serialized_end=1774 + _ROLLSITEPULLCLEARSTATUSRESPONSE._serialized_start=1776 + _ROLLSITEPULLCLEARSTATUSRESPONSE._serialized_end=1809 + _TRANSFERSERVICE._serialized_start=1812 + _TRANSFERSERVICE._serialized_end=2159 +# @@protoc_insertion_point(module_scope) diff --git a/python/eggroll/backport/proto/transfer_pb2.pyi b/python/eggroll/backport/proto/transfer_pb2.pyi new file mode 100644 index 000000000..72bcad3a0 --- /dev/null +++ b/python/eggroll/backport/proto/transfer_pb2.pyi @@ -0,0 +1,312 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file + +Copyright (c) 2019 - now, Eggroll Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import builtins +import collections.abc +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.message +import sys + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing_extensions.final +class TransferHeader(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ID_FIELD_NUMBER: builtins.int + TAG_FIELD_NUMBER: builtins.int + TOTALSIZE_FIELD_NUMBER: builtins.int + STATUS_FIELD_NUMBER: builtins.int + EXT_FIELD_NUMBER: builtins.int + id: builtins.int + tag: builtins.str + totalSize: builtins.int + status: builtins.str + ext: builtins.bytes + def __init__( + self, + *, + id: builtins.int = ..., + tag: builtins.str = ..., + totalSize: builtins.int = ..., + status: builtins.str = ..., + ext: builtins.bytes = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["ext", b"ext", "id", b"id", "status", b"status", "tag", b"tag", "totalSize", b"totalSize"]) -> None: ... + +global___TransferHeader = TransferHeader + +@typing_extensions.final +class TransferBatch(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + HEADER_FIELD_NUMBER: builtins.int + BATCHSIZE_FIELD_NUMBER: builtins.int + DATA_FIELD_NUMBER: builtins.int + @property + def header(self) -> global___TransferHeader: ... + batchSize: builtins.int + data: builtins.bytes + def __init__( + self, + *, + header: global___TransferHeader | None = ..., + batchSize: builtins.int = ..., + data: builtins.bytes = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["header", b"header"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["batchSize", b"batchSize", "data", b"data", "header", b"header"]) -> None: ... + +global___TransferBatch = TransferBatch + +@typing_extensions.final +class RollSiteHeader(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class OptionsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + ROLLSITESESSIONID_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + TAG_FIELD_NUMBER: builtins.int + SRCROLE_FIELD_NUMBER: builtins.int + SRCPARTYID_FIELD_NUMBER: builtins.int + DSTROLE_FIELD_NUMBER: builtins.int + DSTPARTYID_FIELD_NUMBER: builtins.int + DATATYPE_FIELD_NUMBER: builtins.int + OPTIONS_FIELD_NUMBER: builtins.int + TOTALPARTITIONS_FIELD_NUMBER: builtins.int + PARTITIONID_FIELD_NUMBER: builtins.int + TOTALSTREAMS_FIELD_NUMBER: builtins.int + TOTALBATCHES_FIELD_NUMBER: builtins.int + STREAMSEQ_FIELD_NUMBER: builtins.int + BATCHSEQ_FIELD_NUMBER: builtins.int + STAGE_FIELD_NUMBER: builtins.int + rollSiteSessionId: builtins.str + name: builtins.str + tag: builtins.str + srcRole: builtins.str + srcPartyId: builtins.str + dstRole: builtins.str + dstPartyId: builtins.str + dataType: builtins.str + @property + def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: ... + totalPartitions: builtins.int + partitionId: builtins.int + totalStreams: builtins.int + totalBatches: builtins.int + streamSeq: builtins.int + batchSeq: builtins.int + stage: builtins.str + def __init__( + self, + *, + rollSiteSessionId: builtins.str = ..., + name: builtins.str = ..., + tag: builtins.str = ..., + srcRole: builtins.str = ..., + srcPartyId: builtins.str = ..., + dstRole: builtins.str = ..., + dstPartyId: builtins.str = ..., + dataType: builtins.str = ..., + options: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + totalPartitions: builtins.int = ..., + partitionId: builtins.int = ..., + totalStreams: builtins.int = ..., + totalBatches: builtins.int = ..., + streamSeq: builtins.int = ..., + batchSeq: builtins.int = ..., + stage: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["batchSeq", b"batchSeq", "dataType", b"dataType", "dstPartyId", b"dstPartyId", "dstRole", b"dstRole", "name", b"name", "options", b"options", "partitionId", b"partitionId", "rollSiteSessionId", b"rollSiteSessionId", "srcPartyId", b"srcPartyId", "srcRole", b"srcRole", "stage", b"stage", "streamSeq", b"streamSeq", "tag", b"tag", "totalBatches", b"totalBatches", "totalPartitions", b"totalPartitions", "totalStreams", b"totalStreams"]) -> None: ... + +global___RollSiteHeader = RollSiteHeader + +@typing_extensions.final +class RollSitePullGetHeaderRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TAG_FIELD_NUMBER: builtins.int + TIMEOUT_FIELD_NUMBER: builtins.int + tag: builtins.str + timeout: builtins.float + def __init__( + self, + *, + tag: builtins.str = ..., + timeout: builtins.float = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["tag", b"tag", "timeout", b"timeout"]) -> None: ... + +global___RollSitePullGetHeaderRequest = RollSitePullGetHeaderRequest + +@typing_extensions.final +class RollSitePullGetHeaderResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + HEADER_FIELD_NUMBER: builtins.int + @property + def header(self) -> global___RollSiteHeader: ... + def __init__( + self, + *, + header: global___RollSiteHeader | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["header", b"header"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["header", b"header"]) -> None: ... + +global___RollSitePullGetHeaderResponse = RollSitePullGetHeaderResponse + +@typing_extensions.final +class RollSitePullGetPartitionStatusRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TAG_FIELD_NUMBER: builtins.int + TIMEOUT_FIELD_NUMBER: builtins.int + tag: builtins.str + timeout: builtins.float + def __init__( + self, + *, + tag: builtins.str = ..., + timeout: builtins.float = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["tag", b"tag", "timeout", b"timeout"]) -> None: ... + +global___RollSitePullGetPartitionStatusRequest = RollSitePullGetPartitionStatusRequest + +@typing_extensions.final +class RollSitePullGetPartitionStatusResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + @typing_extensions.final + class IntKeyIntValuePair(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.int + value: builtins.int + def __init__( + self, + *, + key: builtins.int = ..., + value: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ... + + @typing_extensions.final + class RollSitePullGetPartitionStatusResponseStatus(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TAG_FIELD_NUMBER: builtins.int + IS_FINISHED_FIELD_NUMBER: builtins.int + TOTAL_BATCHES_FIELD_NUMBER: builtins.int + BATCH_SEQ_TO_PAIR_COUNTER_FIELD_NUMBER: builtins.int + TOTAL_STREAMS_FIELD_NUMBER: builtins.int + STREAM_SEQ_TO_PAIR_COUNTER_FIELD_NUMBER: builtins.int + STREAM_SEQ_TO_BATCH_SEQ_FIELD_NUMBER: builtins.int + TOTAL_PAIRS_FIELD_NUMBER: builtins.int + DATA_TYPE_FIELD_NUMBER: builtins.int + tag: builtins.str + is_finished: builtins.bool + total_batches: builtins.int + @property + def batch_seq_to_pair_counter(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___RollSitePullGetPartitionStatusResponse.IntKeyIntValuePair]: ... + total_streams: builtins.int + @property + def stream_seq_to_pair_counter(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___RollSitePullGetPartitionStatusResponse.IntKeyIntValuePair]: ... + @property + def stream_seq_to_batch_seq(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___RollSitePullGetPartitionStatusResponse.IntKeyIntValuePair]: ... + total_pairs: builtins.int + data_type: builtins.str + def __init__( + self, + *, + tag: builtins.str = ..., + is_finished: builtins.bool = ..., + total_batches: builtins.int = ..., + batch_seq_to_pair_counter: collections.abc.Iterable[global___RollSitePullGetPartitionStatusResponse.IntKeyIntValuePair] | None = ..., + total_streams: builtins.int = ..., + stream_seq_to_pair_counter: collections.abc.Iterable[global___RollSitePullGetPartitionStatusResponse.IntKeyIntValuePair] | None = ..., + stream_seq_to_batch_seq: collections.abc.Iterable[global___RollSitePullGetPartitionStatusResponse.IntKeyIntValuePair] | None = ..., + total_pairs: builtins.int = ..., + data_type: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["batch_seq_to_pair_counter", b"batch_seq_to_pair_counter", "data_type", b"data_type", "is_finished", b"is_finished", "stream_seq_to_batch_seq", b"stream_seq_to_batch_seq", "stream_seq_to_pair_counter", b"stream_seq_to_pair_counter", "tag", b"tag", "total_batches", b"total_batches", "total_pairs", b"total_pairs", "total_streams", b"total_streams"]) -> None: ... + + PARTITION_ID_FIELD_NUMBER: builtins.int + STATUS_FIELD_NUMBER: builtins.int + partition_id: builtins.int + @property + def status(self) -> global___RollSitePullGetPartitionStatusResponse.RollSitePullGetPartitionStatusResponseStatus: ... + def __init__( + self, + *, + partition_id: builtins.int = ..., + status: global___RollSitePullGetPartitionStatusResponse.RollSitePullGetPartitionStatusResponseStatus | None = ..., + ) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["status", b"status"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["partition_id", b"partition_id", "status", b"status"]) -> None: ... + +global___RollSitePullGetPartitionStatusResponse = RollSitePullGetPartitionStatusResponse + +@typing_extensions.final +class RollSitePullClearStatusRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + TAG_FIELD_NUMBER: builtins.int + tag: builtins.str + def __init__( + self, + *, + tag: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["tag", b"tag"]) -> None: ... + +global___RollSitePullClearStatusRequest = RollSitePullClearStatusRequest + +@typing_extensions.final +class RollSitePullClearStatusResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + def __init__( + self, + ) -> None: ... + +global___RollSitePullClearStatusResponse = RollSitePullClearStatusResponse diff --git a/python/eggroll/backport/proto/transfer_pb2_grpc.py b/python/eggroll/backport/proto/transfer_pb2_grpc.py new file mode 100644 index 000000000..394091532 --- /dev/null +++ b/python/eggroll/backport/proto/transfer_pb2_grpc.py @@ -0,0 +1,135 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +import transfer_pb2 as transfer__pb2 + + +class TransferServiceStub(object): + """TODO: use transfer lib + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.send = channel.stream_unary( + '/com.webank.eggroll.core.transfer.TransferService/send', + request_serializer=transfer__pb2.TransferBatch.SerializeToString, + response_deserializer=transfer__pb2.TransferBatch.FromString, + ) + self.recv = channel.unary_stream( + '/com.webank.eggroll.core.transfer.TransferService/recv', + request_serializer=transfer__pb2.TransferBatch.SerializeToString, + response_deserializer=transfer__pb2.TransferBatch.FromString, + ) + self.sendRecv = channel.stream_stream( + '/com.webank.eggroll.core.transfer.TransferService/sendRecv', + request_serializer=transfer__pb2.TransferBatch.SerializeToString, + response_deserializer=transfer__pb2.TransferBatch.FromString, + ) + + +class TransferServiceServicer(object): + """TODO: use transfer lib + """ + + def send(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def recv(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def sendRecv(self, request_iterator, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_TransferServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'send': grpc.stream_unary_rpc_method_handler( + servicer.send, + request_deserializer=transfer__pb2.TransferBatch.FromString, + response_serializer=transfer__pb2.TransferBatch.SerializeToString, + ), + 'recv': grpc.unary_stream_rpc_method_handler( + servicer.recv, + request_deserializer=transfer__pb2.TransferBatch.FromString, + response_serializer=transfer__pb2.TransferBatch.SerializeToString, + ), + 'sendRecv': grpc.stream_stream_rpc_method_handler( + servicer.sendRecv, + request_deserializer=transfer__pb2.TransferBatch.FromString, + response_serializer=transfer__pb2.TransferBatch.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'com.webank.eggroll.core.transfer.TransferService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class TransferService(object): + """TODO: use transfer lib + """ + + @staticmethod + def send(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_unary(request_iterator, target, '/com.webank.eggroll.core.transfer.TransferService/send', + transfer__pb2.TransferBatch.SerializeToString, + transfer__pb2.TransferBatch.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def recv(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/com.webank.eggroll.core.transfer.TransferService/recv', + transfer__pb2.TransferBatch.SerializeToString, + transfer__pb2.TransferBatch.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def sendRecv(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_stream(request_iterator, target, '/com.webank.eggroll.core.transfer.TransferService/sendRecv', + transfer__pb2.TransferBatch.SerializeToString, + transfer__pb2.TransferBatch.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) From cbc2f8afedcd611d0b14057d3a7860e5952863c2 Mon Sep 17 00:00:00 2001 From: sagewe Date: Tue, 16 Jan 2024 13:32:02 +0800 Subject: [PATCH 05/20] feat: enable backport according env (#708) use `EGGROLL_2X_BACKPORT` env to enable backport and python/eggroll/backport directory can safety delete if you don't need this feature. Signed-off-by: sagewe --- python/eggroll/__init__.py | 7 +++++++ python/eggroll/backport/__init__.py | 7 +------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/python/eggroll/__init__.py b/python/eggroll/__init__.py index f19050087..bd13d5775 100644 --- a/python/eggroll/__init__.py +++ b/python/eggroll/__init__.py @@ -15,3 +15,10 @@ # __version__ = "3.0.0" + +import os + +if os.environ.get("EGGROLL_2X_BACKPORT"): + from .backport import backport_patch + + backport_patch() diff --git a/python/eggroll/backport/__init__.py b/python/eggroll/backport/__init__.py index 6817c1bca..2c10a5766 100644 --- a/python/eggroll/backport/__init__.py +++ b/python/eggroll/backport/__init__.py @@ -14,14 +14,9 @@ # # -import sys - def backport_patch(): + import sys from ._package_loader_patch import Eggroll2xPackageFinder sys.meta_path.insert(0, Eggroll2xPackageFinder()) - - -if __name__ == "__main__": - backport_patch() From dfb46003610306854ec7ca8bdc90dfd3de995bf9 Mon Sep 17 00:00:00 2001 From: sagewe Date: Tue, 16 Jan 2024 13:37:56 +0800 Subject: [PATCH 06/20] chore: fix requirements (#708) 1. add some missing requirements to requirements.txt 2. add requirements-2.x.txt for backport situation since protobuf version is different Signed-off-by: sagewe --- requirements-2.x.txt | 11 +++++++++++ requirements.txt | 5 +++++ 2 files changed, 16 insertions(+) create mode 100644 requirements-2.x.txt diff --git a/requirements-2.x.txt b/requirements-2.x.txt new file mode 100644 index 000000000..d7fe5ec0c --- /dev/null +++ b/requirements-2.x.txt @@ -0,0 +1,11 @@ +cloudpickle==2.1.0 +lmdb==1.3.0 +protobuf==3.19.6 +grpcio==1.46.3 +grpcio-tools==1.46.3 +psutil>=5.7.0 +opentelemetry-api +opentelemetry-sdk +omegaconf +requests +mmh3 diff --git a/requirements.txt b/requirements.txt index 0347eb7c9..1c14f2642 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,8 @@ protobuf==4.24.4 grpcio==1.59.3 grpcio-tools==1.59.3 psutil>=5.7.0 +opentelemetry-api +opentelemetry-sdk +omegaconf +requests + From 863342200d98f94b20fc953c3a4c3e163866dd73 Mon Sep 17 00:00:00 2001 From: sagewe Date: Tue, 16 Jan 2024 13:48:21 +0800 Subject: [PATCH 07/20] refactor: rollback modification in federation (#708) Signed-off-by: sagewe --- python/eggroll/backport/_wrap_rollsite.py | 32 ++++++++++++++++++++--- python/eggroll/federation/__init__.py | 3 +-- python/eggroll/federation/_rollsite.py | 20 -------------- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/python/eggroll/backport/_wrap_rollsite.py b/python/eggroll/backport/_wrap_rollsite.py index 31590629d..fcad0e5f6 100644 --- a/python/eggroll/backport/_wrap_rollsite.py +++ b/python/eggroll/backport/_wrap_rollsite.py @@ -17,7 +17,7 @@ import typing if typing.TYPE_CHECKING: - from eggroll.federation import RollSite + from eggroll.federation._rollsite import RollSite from ._wrap_rollpair import WrappedRpc @@ -59,7 +59,7 @@ def load(self, name, tag, options: dict = None): options = {} final_options = self._options.copy() final_options.update(options) - from eggroll.federation import RollSite + from eggroll.federation._rollsite import RollSite return WrappedRollSite( RollSite(name=name, tag=tag, rs_ctx=self._rsc, options=final_options) @@ -80,7 +80,33 @@ def push(self, obj, parties: list = None, options: dict = None): return self._roll_site.push_bytes(Serdes.serialize(obj), parties, options) def pull(self, parties: list = None, options: dict = None): - return self._roll_site.pull_with_lift(lifter=_lifter, parties=parties) + return _pull_with_lift(self._roll_site, parties=parties) + + +def _pull_with_lift( + rs: "RollSite", + parties: list = None, +): + from eggroll.federation._rollsite import ErRollSiteHeader + + futures = [] + for src_role, src_party_id in parties: + src_party_id = str(src_party_id) + rs_header = ErRollSiteHeader( + roll_site_session_id=rs.roll_site_session_id, + name=rs.name, + tag=rs.tag, + src_role=src_role, + src_party_id=src_party_id, + dst_role=rs.local_role, + dst_party_id=rs.party_id, + ) + futures.append( + rs._receive_executor_pool.submit( + _lifter(rs._impl_instance._pull_one), rs_header + ) + ) + return futures def _lifter(func): diff --git a/python/eggroll/federation/__init__.py b/python/eggroll/federation/__init__.py index 259342c4f..421c0f0b4 100644 --- a/python/eggroll/federation/__init__.py +++ b/python/eggroll/federation/__init__.py @@ -16,6 +16,5 @@ from ._rollsite_context import RollSiteContext -from ._rollsite import RollSite -__all__ = ["RollSiteContext", "RollSite"] +__all__ = ["RollSiteContext"] diff --git a/python/eggroll/federation/_rollsite.py b/python/eggroll/federation/_rollsite.py index 0de4637a0..ca7217747 100644 --- a/python/eggroll/federation/_rollsite.py +++ b/python/eggroll/federation/_rollsite.py @@ -110,23 +110,3 @@ def pull(self, parties: list = None): ) ) return futures - - def pull_with_lift(self, lifter: typing.Callable[[typing.Callable], typing.Callable], parties: list = None): - futures = [] - for src_role, src_party_id in parties: - src_party_id = str(src_party_id) - rs_header = ErRollSiteHeader( - roll_site_session_id=self.roll_site_session_id, - name=self.name, - tag=self.tag, - src_role=src_role, - src_party_id=src_party_id, - dst_role=self.local_role, - dst_party_id=self.party_id, - ) - futures.append( - self._receive_executor_pool.submit( - lifter(self._impl_instance._pull_one), rs_header - ) - ) - return futures From a03756a813c5c38c4d516e0c954d1da5c4a3535d Mon Sep 17 00:00:00 2001 From: sagewe Date: Tue, 16 Jan 2024 14:21:55 +0800 Subject: [PATCH 08/20] fix: fix import and api related to flow use case (#708) Signed-off-by: sagewe --- .../backport/_module_rollpair_patch.py | 8 ++++-- python/eggroll/backport/_wrap_rollpair.py | 25 ++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/python/eggroll/backport/_module_rollpair_patch.py b/python/eggroll/backport/_module_rollpair_patch.py index 856f02056..549dd23e2 100644 --- a/python/eggroll/backport/_module_rollpair_patch.py +++ b/python/eggroll/backport/_module_rollpair_patch.py @@ -14,6 +14,10 @@ # # -from ._wrap_rollpair import WrappedRp as RollPair, runtime_init +from ._wrap_rollpair import ( + WrappedRp as RollPair, + WrappedRpc as RollPairContext, + runtime_init, +) -__all__ = ["RollPair", "runtime_init"] +__all__ = ["RollPair", "RollPairContext", "runtime_init"] diff --git a/python/eggroll/backport/_wrap_rollpair.py b/python/eggroll/backport/_wrap_rollpair.py index 58b4640ac..1cb394ff7 100644 --- a/python/eggroll/backport/_wrap_rollpair.py +++ b/python/eggroll/backport/_wrap_rollpair.py @@ -33,8 +33,13 @@ def runtime_init(session: "WrappedSession") -> "WrappedRpc": class WrappedRpc: - def __init__(self, rpc: "RollPairContext"): - self._rpc = rpc + def __init__(self, rpc: "RollPairContext" = None, session: "WrappedSession" = None): + if rpc is not None: + self._rpc = rpc + if session is not None: + from eggroll.computing import runtime_init as _runtime_init + + self._rpc = _runtime_init(session=session._session) @property def session_id(self): @@ -44,7 +49,19 @@ def load(self, namespace, name, options): from eggroll.computing.tasks.store import StoreTypes store_type = options.get("store_type", StoreTypes.ROLLPAIR_LMDB) - return self._rpc.load_rp(namespace=namespace, name=name, store_type=store_type) + return WrappedRp( + self._rpc.create_rp( + id=-1, + name=name, + namespace=namespace, + total_partitions=1, + store_type=store_type, + key_serdes_type=0, + value_serdes_type=0, + partitioner_type=0, + options=options, + ) + ) def parallelize(self, data, options: dict = None): from eggroll.computing.tasks.store import StoreTypes @@ -150,6 +167,7 @@ def put_all(self, items, output=None, options: dict = None): if include_key: self._rp.put_all( ((Serdes.serialize(k), Serdes.serialize(v)) for k, v in items), + partitioner=mmh3_partitioner, ) else: self._rp.put_all( @@ -157,6 +175,7 @@ def put_all(self, items, output=None, options: dict = None): (Serdes.serialize(i), Serdes.serialize(v)) for i, v in enumerate(items) ), + partitioner=mmh3_partitioner, ) def take(self, n: int, options: dict = None): From 3b438902457943153714f9a0b94bd7db5c95e76d Mon Sep 17 00:00:00 2001 From: sagewe Date: Tue, 16 Jan 2024 16:43:50 +0800 Subject: [PATCH 09/20] fix: remove duplicate get_or_create_store call (#708) Signed-off-by: sagewe --- .../eggroll/computing/tasks/impl/_map_partitions_with_index.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/eggroll/computing/tasks/impl/_map_partitions_with_index.py b/python/eggroll/computing/tasks/impl/_map_partitions_with_index.py index 37240ad81..a3d46bb7e 100644 --- a/python/eggroll/computing/tasks/impl/_map_partitions_with_index.py +++ b/python/eggroll/computing/tasks/impl/_map_partitions_with_index.py @@ -89,7 +89,7 @@ def submit( session_id=rp.session_id, tag=consts.MAP_REDUCE_PARTITIONS_WITH_INDEX ) output_store = rp.ctx.create_store( - id=rp.get_store().store_locator.id, + id=-1, name=output_name, namespace=output_namespace, total_partitions=output_num_partitions, @@ -99,7 +99,6 @@ def submit( partitioner_type=output_partitioner_type, options={}, ) - output_store = rp.ctx.session.get_or_create_store(output_store) job = ErJob( id=job_id, name=consts.MAP_REDUCE_PARTITIONS_WITH_INDEX, From 464f5387d863369b972cd51371f57d6741c83ccc Mon Sep 17 00:00:00 2001 From: sagewe Date: Tue, 16 Jan 2024 16:44:43 +0800 Subject: [PATCH 10/20] fix: load should get partition num from option (#708) Signed-off-by: sagewe --- python/eggroll/backport/_wrap_rollpair.py | 64 ++++++++++++++--------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/python/eggroll/backport/_wrap_rollpair.py b/python/eggroll/backport/_wrap_rollpair.py index 1cb394ff7..ea379c1ec 100644 --- a/python/eggroll/backport/_wrap_rollpair.py +++ b/python/eggroll/backport/_wrap_rollpair.py @@ -49,12 +49,13 @@ def load(self, namespace, name, options): from eggroll.computing.tasks.store import StoreTypes store_type = options.get("store_type", StoreTypes.ROLLPAIR_LMDB) + total_partitions = options.get("total_partitions", 1) return WrappedRp( self._rpc.create_rp( id=-1, name=name, namespace=namespace, - total_partitions=1, + total_partitions=total_partitions, store_type=store_type, key_serdes_type=0, value_serdes_type=0, @@ -130,6 +131,21 @@ def get_name(self): def get_partitions(self): return self._rp.get_partitions() + def save_as(self, name, namespace, partition=None, options: dict = None): + if partition is not None and partition <= 0: + raise ValueError('partition cannot <= 0') + + if partition is not None and partition != self.num_partitions: + repartitioned = self.repartition(num_partitions=partition) + return repartitioned.save_as(name, namespace, options=options) + + if options is None: + options = {} + + from eggroll.computing.tasks.store import StoreTypes + store_type = options.get("store_type", StoreTypes.ROLLPAIR_LMDB) + return WrappedRp(self._rp.copy_as(name=name, namespace=namespace, store_type=store_type)) + def get(self, k, options: dict = None): if options is None: options = {} @@ -185,7 +201,7 @@ def take(self, n: int, options: dict = None): else: return [ (Serdes.deserialize(k), Serdes.deserialize(v)) - for k, v in self._rp.take(n=n, options=options) + for k, v in self._rp.take(num=n, options=options) ] def first(self, options: dict = None): @@ -199,11 +215,11 @@ def destroy(self, options: dict = None): self._rp.destroy() def _map_reduce_partitions_with_index( - self, - map_partition_op: Callable[[int, Iterable], Iterable], - reduce_partition_op: Callable[[Any, Any], Any] = None, - shuffle=True, - output_num_partitions=None, + self, + map_partition_op: Callable[[int, Iterable], Iterable], + reduce_partition_op: Callable[[Any, Any], Any] = None, + shuffle=True, + output_num_partitions=None, ): if not shuffle and reduce_partition_op is not None: raise ValueError( @@ -359,11 +375,11 @@ def repartition_with(self, other: "WrappedRp") -> Tuple["WrappedRp", "WrappedRp" return self.repartition(other.num_partitions), other def binarySortedMapPartitionsWithIndex( - self, - other: "WrappedRp", - binary_sorted_map_partitions_with_index_op: Callable[ - [int, Iterable, Iterable], Iterable - ], + self, + other: "WrappedRp", + binary_sorted_map_partitions_with_index_op: Callable[ + [int, Iterable, Iterable], Iterable + ], ): first, second = self.repartition_with(other) @@ -420,12 +436,12 @@ def join(self, other, func, output=None, options: dict = None): def _lifted_map_to_io_serdes( - _f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes + _f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes ): def _lifted(_index, _iter): for out_k, out_v in _f( - _index, - _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes), + _index, + _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes), ): yield output_key_serdes.serialize(out_k), output_value_serdes.serialize( out_v @@ -445,12 +461,12 @@ def _value_serdes_wrapped_generator(_iter, value_serdes): def _lifted_mpwi_map_to_serdes( - _f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes + _f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes ): def _lifted(_index, _iter): for out_k, out_v in _f( - _index, - _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes), + _index, + _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes), ): yield output_key_serdes.serialize(out_k), output_value_serdes.serialize( out_v @@ -491,7 +507,7 @@ def _lifted(_index, _iter): def _lifted_map_reduce_partitions_to_mpwi( - map_partition_op: Callable[[Iterable], Iterable] + map_partition_op: Callable[[Iterable], Iterable] ): def _lifted(_index, _iter): return map_partition_op(_iter) @@ -530,7 +546,7 @@ def _lifted(_index, _iter): def _lifted_flat_map_to_mpwi( - flat_map_op: Callable[[Any, Any], Iterable[Tuple[Any, Any]]] + flat_map_op: Callable[[Any, Any], Iterable[Tuple[Any, Any]]] ): def _lifted(_index, _iter): for _k, _v in _iter: @@ -565,13 +581,13 @@ def _lifted(x, y): def _lifted_sorted_binary_map_partitions_with_index_to_serdes( - _f, left_value_serdes, right_value_serdes, output_value_serdes + _f, left_value_serdes, right_value_serdes, output_value_serdes ): def _lifted(_index, left_iter, right_iter): for out_k_bytes, out_v in _f( - _index, - _value_serdes_wrapped_generator(left_iter, left_value_serdes), - _value_serdes_wrapped_generator(right_iter, right_value_serdes), + _index, + _value_serdes_wrapped_generator(left_iter, left_value_serdes), + _value_serdes_wrapped_generator(right_iter, right_value_serdes), ): yield out_k_bytes, output_value_serdes.serialize(out_v) From ef5a03a988aac148f26f37beb7e93dae3460d5ed Mon Sep 17 00:00:00 2001 From: sagewe Date: Tue, 16 Jan 2024 17:43:34 +0800 Subject: [PATCH 11/20] fix: fix filter(#708) Signed-off-by: sagewe --- python/eggroll/backport/_wrap_rollpair.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/eggroll/backport/_wrap_rollpair.py b/python/eggroll/backport/_wrap_rollpair.py index ea379c1ec..10c276de4 100644 --- a/python/eggroll/backport/_wrap_rollpair.py +++ b/python/eggroll/backport/_wrap_rollpair.py @@ -333,7 +333,7 @@ def filter(self, func, output=None, options: dict = None): if options is None: options = {} return self._map_reduce_partitions_with_index( - lambda i, x: ((k, v) for k, v in x if func(v)), + lambda i, x: ((k, v) for k, v in x if func(k, v)), shuffle=False, ) From d3a7f8713c1cc7f73a6d04fd646028f2a207f1e3 Mon Sep 17 00:00:00 2001 From: sagewe Date: Tue, 16 Jan 2024 17:43:46 +0800 Subject: [PATCH 12/20] fix: fix session stop (#708) Signed-off-by: sagewe --- python/eggroll/backport/_wrap_session.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/eggroll/backport/_wrap_session.py b/python/eggroll/backport/_wrap_session.py index 271a37fda..4e7689fb6 100644 --- a/python/eggroll/backport/_wrap_session.py +++ b/python/eggroll/backport/_wrap_session.py @@ -42,3 +42,9 @@ def __init__(self, session: "ErSession"): def get_session_id(self): return self._session.get_session_id() + + def kill(self): + return self._session.kill() + + def stop(self): + return self._session.stop() From 53e34bb06538bd9ca38f14e837a9c99fbc85d431 Mon Sep 17 00:00:00 2001 From: sagewe Date: Tue, 16 Jan 2024 17:54:16 +0800 Subject: [PATCH 13/20] fix: enable rollpair serialize check again, with switch option (#708) Signed-off-by: sagewe --- python/eggroll/backport/_wrap_rollpair.py | 55 ++++++++++--------- .../eggroll/computing/roll_pair/__init__.py | 4 +- .../eggroll/computing/roll_pair/_roll_pair.py | 11 ++-- .../computing/roll_pair/_roll_pair_context.py | 3 +- 4 files changed, 39 insertions(+), 34 deletions(-) diff --git a/python/eggroll/backport/_wrap_rollpair.py b/python/eggroll/backport/_wrap_rollpair.py index 10c276de4..5959311fa 100644 --- a/python/eggroll/backport/_wrap_rollpair.py +++ b/python/eggroll/backport/_wrap_rollpair.py @@ -28,7 +28,7 @@ def runtime_init(session: "WrappedSession") -> "WrappedRpc": from eggroll.computing import runtime_init as _runtime_init - rpc = _runtime_init(session=session._session) + rpc = _runtime_init(session=session._session, allow_rp_serialize=True) return WrappedRpc(rpc=rpc) @@ -39,7 +39,7 @@ def __init__(self, rpc: "RollPairContext" = None, session: "WrappedSession" = No if session is not None: from eggroll.computing import runtime_init as _runtime_init - self._rpc = _runtime_init(session=session._session) + self._rpc = _runtime_init(session=session._session, allow_rp_serialize=True) @property def session_id(self): @@ -133,7 +133,7 @@ def get_partitions(self): def save_as(self, name, namespace, partition=None, options: dict = None): if partition is not None and partition <= 0: - raise ValueError('partition cannot <= 0') + raise ValueError("partition cannot <= 0") if partition is not None and partition != self.num_partitions: repartitioned = self.repartition(num_partitions=partition) @@ -143,8 +143,11 @@ def save_as(self, name, namespace, partition=None, options: dict = None): options = {} from eggroll.computing.tasks.store import StoreTypes + store_type = options.get("store_type", StoreTypes.ROLLPAIR_LMDB) - return WrappedRp(self._rp.copy_as(name=name, namespace=namespace, store_type=store_type)) + return WrappedRp( + self._rp.copy_as(name=name, namespace=namespace, store_type=store_type) + ) def get(self, k, options: dict = None): if options is None: @@ -215,11 +218,11 @@ def destroy(self, options: dict = None): self._rp.destroy() def _map_reduce_partitions_with_index( - self, - map_partition_op: Callable[[int, Iterable], Iterable], - reduce_partition_op: Callable[[Any, Any], Any] = None, - shuffle=True, - output_num_partitions=None, + self, + map_partition_op: Callable[[int, Iterable], Iterable], + reduce_partition_op: Callable[[Any, Any], Any] = None, + shuffle=True, + output_num_partitions=None, ): if not shuffle and reduce_partition_op is not None: raise ValueError( @@ -375,11 +378,11 @@ def repartition_with(self, other: "WrappedRp") -> Tuple["WrappedRp", "WrappedRp" return self.repartition(other.num_partitions), other def binarySortedMapPartitionsWithIndex( - self, - other: "WrappedRp", - binary_sorted_map_partitions_with_index_op: Callable[ - [int, Iterable, Iterable], Iterable - ], + self, + other: "WrappedRp", + binary_sorted_map_partitions_with_index_op: Callable[ + [int, Iterable, Iterable], Iterable + ], ): first, second = self.repartition_with(other) @@ -436,12 +439,12 @@ def join(self, other, func, output=None, options: dict = None): def _lifted_map_to_io_serdes( - _f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes + _f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes ): def _lifted(_index, _iter): for out_k, out_v in _f( - _index, - _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes), + _index, + _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes), ): yield output_key_serdes.serialize(out_k), output_value_serdes.serialize( out_v @@ -461,12 +464,12 @@ def _value_serdes_wrapped_generator(_iter, value_serdes): def _lifted_mpwi_map_to_serdes( - _f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes + _f, input_key_serdes, input_value_serdes, output_key_serdes, output_value_serdes ): def _lifted(_index, _iter): for out_k, out_v in _f( - _index, - _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes), + _index, + _serdes_wrapped_generator(_iter, input_key_serdes, input_value_serdes), ): yield output_key_serdes.serialize(out_k), output_value_serdes.serialize( out_v @@ -507,7 +510,7 @@ def _lifted(_index, _iter): def _lifted_map_reduce_partitions_to_mpwi( - map_partition_op: Callable[[Iterable], Iterable] + map_partition_op: Callable[[Iterable], Iterable] ): def _lifted(_index, _iter): return map_partition_op(_iter) @@ -546,7 +549,7 @@ def _lifted(_index, _iter): def _lifted_flat_map_to_mpwi( - flat_map_op: Callable[[Any, Any], Iterable[Tuple[Any, Any]]] + flat_map_op: Callable[[Any, Any], Iterable[Tuple[Any, Any]]] ): def _lifted(_index, _iter): for _k, _v in _iter: @@ -581,13 +584,13 @@ def _lifted(x, y): def _lifted_sorted_binary_map_partitions_with_index_to_serdes( - _f, left_value_serdes, right_value_serdes, output_value_serdes + _f, left_value_serdes, right_value_serdes, output_value_serdes ): def _lifted(_index, left_iter, right_iter): for out_k_bytes, out_v in _f( - _index, - _value_serdes_wrapped_generator(left_iter, left_value_serdes), - _value_serdes_wrapped_generator(right_iter, right_value_serdes), + _index, + _value_serdes_wrapped_generator(left_iter, left_value_serdes), + _value_serdes_wrapped_generator(right_iter, right_value_serdes), ): yield out_k_bytes, output_value_serdes.serialize(out_v) diff --git a/python/eggroll/computing/roll_pair/__init__.py b/python/eggroll/computing/roll_pair/__init__.py index b1580dbce..e163f4bc0 100644 --- a/python/eggroll/computing/roll_pair/__init__.py +++ b/python/eggroll/computing/roll_pair/__init__.py @@ -2,8 +2,8 @@ from ._roll_pair import RollPair -def runtime_init(session: ErSession): - rpc = RollPairContext(session=session) +def runtime_init(session: ErSession, allow_rp_serialize=False): + rpc = RollPairContext(session=session, allow_rp_serialize=allow_rp_serialize) return rpc diff --git a/python/eggroll/computing/roll_pair/_roll_pair.py b/python/eggroll/computing/roll_pair/_roll_pair.py index 00d351cfd..da6a69570 100644 --- a/python/eggroll/computing/roll_pair/_roll_pair.py +++ b/python/eggroll/computing/roll_pair/_roll_pair.py @@ -27,7 +27,6 @@ if typing.TYPE_CHECKING: from ._roll_pair_context import RollPairContext - L = logging.getLogger(__name__) T = typing.TypeVar("T") @@ -35,10 +34,12 @@ class RollPair(object): def __getstate__(self): - return None - # raise NotImplementedError( - # "pickling RollPair is not expected behavior, if you really need it, please create an issue at" - # ) + if self._ctx._allow_rp_serialize: + return None + else: + raise NotImplementedError( + "pickling RollPair is not expected behavior, if you really need it, please create an issue at" + ) def __init__(self, er_store: ErStore, rp_ctx: "RollPairContext", gc_enabled=None): self._ctx = rp_ctx diff --git a/python/eggroll/computing/roll_pair/_roll_pair_context.py b/python/eggroll/computing/roll_pair/_roll_pair_context.py index c3f53051e..736f0277e 100644 --- a/python/eggroll/computing/roll_pair/_roll_pair_context.py +++ b/python/eggroll/computing/roll_pair/_roll_pair_context.py @@ -33,7 +33,7 @@ class RollPairContext(object): - def __init__(self, session: ErSession): + def __init__(self, session: ErSession, allow_rp_serialize=False): if not session.is_active(): raise Exception( f"session_id={session.get_session_id()} is not ACTIVE. current status={session.get_session_meta().status}" @@ -43,6 +43,7 @@ def __init__(self, session: ErSession): self._rpc_gc_enable = True self._command_client = CommandClient(config=session.config) self._session.add_exit_task(self._gc_recorder.flush) + self._allow_rp_serialize = allow_rp_serialize def info(self, level=0): if level == 0: From a96c65c92c439fe1d4a5594b204931ecaed44921 Mon Sep 17 00:00:00 2001 From: sagewe Date: Tue, 16 Jan 2024 18:08:48 +0800 Subject: [PATCH 14/20] fix: fix corner case when allow_rp_serialize enabled (#708) Signed-off-by: sagewe --- python/eggroll/computing/roll_pair/_roll_pair.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/eggroll/computing/roll_pair/_roll_pair.py b/python/eggroll/computing/roll_pair/_roll_pair.py index da6a69570..27de66e45 100644 --- a/python/eggroll/computing/roll_pair/_roll_pair.py +++ b/python/eggroll/computing/roll_pair/_roll_pair.py @@ -34,7 +34,7 @@ class RollPair(object): def __getstate__(self): - if self._ctx._allow_rp_serialize: + if not hasattr(self, "_ctx") or self._ctx._allow_rp_serialize: return None else: raise NotImplementedError( From 10ba19615cc9667b67eabebb0da1c43690e54522 Mon Sep 17 00:00:00 2001 From: v_wbxiongli <740332065@qq.com> Date: Mon, 26 Feb 2024 15:26:07 +0800 Subject: [PATCH 15/20] update get gpu count Signed-off-by: v_wbxiongli <740332065@qq.com> --- bin/gpu/nvidia.sh | 2 +- python/eggroll/deepspeed/utils/gpu_count.py | 11 +++++++++++ requirements.txt | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 python/eggroll/deepspeed/utils/gpu_count.py diff --git a/bin/gpu/nvidia.sh b/bin/gpu/nvidia.sh index 78432fbd4..5ec7911a5 100644 --- a/bin/gpu/nvidia.sh +++ b/bin/gpu/nvidia.sh @@ -1,3 +1,3 @@ -result=$(nvidia-smi --query-gpu=name --format=csv, noheader|grep 'NVIDIA'|wc -l) +result=$(python $EGGROLL_HOME/python/eggroll/deepspeed/utils/gpu_count.py) echo $result diff --git a/python/eggroll/deepspeed/utils/gpu_count.py b/python/eggroll/deepspeed/utils/gpu_count.py new file mode 100644 index 000000000..a29e3817e --- /dev/null +++ b/python/eggroll/deepspeed/utils/gpu_count.py @@ -0,0 +1,11 @@ +import pynvml + +def gpu_count(): + try: + pynvml.nvmlInit() + return pynvml.nvmlDeviceGetCount() + except Exception as e: + return 0 + +if __name__ == '__main__': + print(gpu_count()) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 0347eb7c9..fbd6c3836 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ protobuf==4.24.4 grpcio==1.59.3 grpcio-tools==1.59.3 psutil>=5.7.0 +pynvml==11.5.0 \ No newline at end of file From 5d50be9e2cf25f6bf58add31deecf3493cca50d2 Mon Sep 17 00:00:00 2001 From: sagewe Date: Tue, 27 Feb 2024 17:42:26 +0800 Subject: [PATCH 16/20] feat: add whitelist serde (#708) Signed-off-by: sagewe --- python/eggroll/backport/_serdes.py | 85 ++++++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 10 deletions(-) diff --git a/python/eggroll/backport/_serdes.py b/python/eggroll/backport/_serdes.py index 50aa3e1b6..71c7c7a95 100644 --- a/python/eggroll/backport/_serdes.py +++ b/python/eggroll/backport/_serdes.py @@ -14,18 +14,83 @@ # # -from pickle import dumps as p_dumps -from pickle import loads as p_loads +import importlib +from pickle import Unpickler, UnpicklingError +# from pickle import dumps as p_dumps +# from pickle import loads as p_loads -class UnrestrictedSerdes: - @staticmethod - def serialize(obj) -> bytes: - return p_dumps(obj) +# class UnrestrictedSerdes: +# @staticmethod +# def serialize(obj) -> bytes: +# return p_dumps(obj) +# +# @staticmethod +# def deserialize(bytes) -> object: +# return p_loads(bytes) + + +class RestrictedUnpickler(Unpickler): + + def _load(self, module, name): + try: + return super().find_class(module, name) + except: + return getattr(importlib.import_module(module), name) + + def find_class(self, module, name): + if name in _DeserializeWhitelist.get_whitelist().get(module, set()): + return self._load(module, name) + else: + for m in _DeserializeWhitelist.get_whitelist_glob(): + if module.startswith(m): + return self._load(module, name) + raise UnpicklingError(f"forbidden unpickle class {module} {name}") + + +class _DeserializeWhitelist: + loaded = False + deserialize_whitelist = {} + deserialize_glob_whitelist = set() + + @classmethod + def get_whitelist_glob(cls): + if not cls.loaded: + cls.load_deserialize_whitelist() + return cls.deserialize_glob_whitelist + + @classmethod + def get_whitelist(cls): + if not cls.loaded: + cls.load_deserialize_whitelist() + return cls.deserialize_whitelist + + @classmethod + def get_whitelist_path(cls): + import os.path + + return os.path.abspath( + os.path.join( + __file__, + os.path.pardir, + os.path.pardir, + os.path.pardir, + os.path.pardir, + "conf", + "whitelist.json", + ) + ) - @staticmethod - def deserialize(bytes) -> object: - return p_loads(bytes) + @classmethod + def load_deserialize_whitelist(cls): + import json + with open(cls.get_whitelist_path()) as f: + for k, v in json.load(f).items(): + if k.endswith("*"): + cls.deserialize_glob_whitelist.add(k[:-1]) + else: + cls.deserialize_whitelist[k] = set(v) + cls.loaded = True -Serdes = UnrestrictedSerdes +Serdes = RestrictedUnpickler From 1da03cf88c0f0c253117793e64bcb6be2f60e476 Mon Sep 17 00:00:00 2001 From: Xiongli <740332065@qq.com> Date: Tue, 27 Feb 2024 19:55:04 +0800 Subject: [PATCH 17/20] update version 3.1.0 Signed-off-by: Xiongli <740332065@qq.com> --- ...75\262\346\226\207\346\241\243\350\257\264\346\230\216.md" | 2 +- doc/upgrade_helper_guide.md | 2 +- jvm/cluster_dashboard/pom.xml | 2 +- jvm/cluster_manager/pom.xml | 2 +- jvm/cluster_manager/src/test/resources/application.properties | 2 +- jvm/core/pom.xml | 2 +- jvm/node_manager/pom.xml | 2 +- jvm/pom.xml | 4 ++-- python/eggroll/__init__.py | 2 +- python/setup.py | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git "a/doc/Eggroll\351\203\250\347\275\262\346\226\207\346\241\243\350\257\264\346\230\216.md" "b/doc/Eggroll\351\203\250\347\275\262\346\226\207\346\241\243\350\257\264\346\230\216.md" index 580928bc1..05342e429 100644 --- "a/doc/Eggroll\351\203\250\347\275\262\346\226\207\346\241\243\350\257\264\346\230\216.md" +++ "b/doc/Eggroll\351\203\250\347\275\262\346\226\207\346\241\243\350\257\264\346\230\216.md" @@ -21,7 +21,7 @@ 从github拉取Eggroll项目，通过执行auto-packaging.sh自动打包脚本在同目录下生成eggroll.tar.gz ```shell -git clone -b v3.0.0 https://github.com/FederatedAI/eggroll.git +git clone -b v3.1.0 https://github.com/FederatedAI/eggroll.git cd eggroll/deploy windows : auto-packaging.bat linux : sh auto-packaging.sh diff --git a/doc/upgrade_helper_guide.md b/doc/upgrade_helper_guide.md index 87f415354..758cc20e0 100644 --- a/doc/upgrade_helper_guide.md +++ b/doc/upgrade_helper_guide.md @@ -89,7 +89,7 @@ ${MYSQL_HOME_PATH}/bin/mysqldump -h -u -p -P 获取升级脚本 -[升级脚本](https://github.com/WeBankFinTech/eggroll/blob/dev-3.0.0-rc/deploy/upgrade_helper.py) +[升级脚本](https://github.com/WeBankFinTech/eggroll/blob/dev-3.1.0/deploy/upgrade_helper.py) ``` export PKG_BASE_PATH={your upgrade eggroll version eggroll home dir} diff --git a/jvm/cluster_dashboard/pom.xml b/jvm/cluster_dashboard/pom.xml index 84203daf4..878d245d4 100644 --- a/jvm/cluster_dashboard/pom.xml +++ b/jvm/cluster_dashboard/pom.xml @@ -6,7 +6,7 @@ eggroll-all org.fedai.eggroll ../pom.xml - 3.0.0 + 3.1.0 4.0.0 cluster_dashboard diff --git a/jvm/cluster_manager/pom.xml b/jvm/cluster_manager/pom.xml index be4dd1a29..b143ae38d 100644 --- a/jvm/cluster_manager/pom.xml +++ b/jvm/cluster_manager/pom.xml @@ -6,7 +6,7 @@ org.fedai.eggroll eggroll-all ../pom.xml - 3.0.0 + 3.1.0 4.0.0 cluster_manager diff --git a/jvm/cluster_manager/src/test/resources/application.properties b/jvm/cluster_manager/src/test/resources/application.properties index d6941b1e5..a83479d2c 100644 --- a/jvm/cluster_manager/src/test/resources/application.properties +++ b/jvm/cluster_manager/src/test/resources/application.properties @@ -16,7 +16,7 @@ [eggroll] application.title=eggroll -application.version=3.0.0 +application.version=3.1.0 # core #eggroll.resourcemanager.clustermanager.jdbc.driver.class.name=org.h2.Driver eggroll.resourcemanager.clustermanager.jdbc.driver.class.name=com.mysql.cj.jdbc.Driver diff --git a/jvm/core/pom.xml b/jvm/core/pom.xml index 1208852f5..479f1c785 100644 --- a/jvm/core/pom.xml +++ b/jvm/core/pom.xml @@ -6,7 +6,7 @@ org.fedai.eggroll eggroll-all ../pom.xml - 3.0.0 + 3.1.0 4.0.0 core diff --git a/jvm/node_manager/pom.xml b/jvm/node_manager/pom.xml index cbd3ee19c..929773535 100644 --- a/jvm/node_manager/pom.xml +++ b/jvm/node_manager/pom.xml @@ -6,7 +6,7 @@ eggroll-all org.fedai.eggroll ../pom.xml - 3.0.0 + 3.1.0 4.0.0 node_manager diff --git a/jvm/pom.xml b/jvm/pom.xml index 2fa8dba23..418686e36 100644 --- a/jvm/pom.xml +++ b/jvm/pom.xml @@ -23,7 +23,7 @@ org.fedai.eggroll eggroll-all - 3.0.0 + 3.1.0 Eggroll Project @@ -145,7 +145,7 @@ 4.0.0 - 3.0.0 + 3.1.0 512m diff --git a/python/eggroll/__init__.py b/python/eggroll/__init__.py index f19050087..b1b934dbd 100644 --- a/python/eggroll/__init__.py +++ b/python/eggroll/__init__.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "3.0.0" +__version__ = "3.1.0" diff --git a/python/setup.py b/python/setup.py index 68571a409..372fc51b7 100644 --- a/python/setup.py +++ b/python/setup.py @@ -43,7 +43,7 @@ setup_kwargs = { "name": "eggroll", - "version": "3.0.0", + "version": "3.1.0", "description": "Python Side Client and Server for Eggroll", "long_description_content_type": "text/markdown", "long_description": "Python Side Client and Server for Eggroll", From 87eddc6505fa1cb573e68463df6a3cbee5a6f5db Mon Sep 17 00:00:00 2001 From: v_wbxiongli <740332065@qq.com> Date: Wed, 28 Feb 2024 17:26:54 +0800 Subject: [PATCH 18/20] update BUILD_INFO version Signed-off-by: v_wbxiongli <740332065@qq.com> --- BUILD_INFO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD_INFO b/BUILD_INFO index deb49b778..f63000821 100644 --- a/BUILD_INFO +++ b/BUILD_INFO @@ -1 +1 @@ -eggroll.version=3.0.0 +eggroll.version=3.1.0 From 9db7e71e573f526de02e59414fab0cce1855226e Mon Sep 17 00:00:00 2001 From: sagewe Date: Wed, 28 Feb 2024 18:50:35 +0800 Subject: [PATCH 19/20] fix: fix restricted serde (#708) Signed-off-by: sagewe --- python/eggroll/backport/_serdes.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/python/eggroll/backport/_serdes.py b/python/eggroll/backport/_serdes.py index 71c7c7a95..65c11b153 100644 --- a/python/eggroll/backport/_serdes.py +++ b/python/eggroll/backport/_serdes.py @@ -16,7 +16,9 @@ import importlib from pickle import Unpickler, UnpicklingError -# from pickle import dumps as p_dumps +from pickle import dumps as p_dumps + + # from pickle import loads as p_loads @@ -29,6 +31,15 @@ # def deserialize(bytes) -> object: # return p_loads(bytes) +class RestrictedSerdes: + @staticmethod + def serialize(obj) -> bytes: + return p_dumps(obj) + + @staticmethod + def deserialize(bytes) -> object: + return RestrictedUnpickler(bytes).load() + class RestrictedUnpickler(Unpickler): @@ -93,4 +104,4 @@ def load_deserialize_whitelist(cls): cls.loaded = True -Serdes = RestrictedUnpickler +Serdes = RestrictedSerdes From b99d3c9c1cb7c850779d1a645d22c5498a1184a6 Mon Sep 17 00:00:00 2001 From: sagewe Date: Thu, 29 Feb 2024 10:49:56 +0800 Subject: [PATCH 20/20] fix: add docs and whitelist (#708) Signed-off-by: sagewe --- conf/whitelist.json | 138 +++++++++++++++++++++++++++++ python/eggroll/backport/README.md | 5 ++ python/eggroll/backport/_serdes.py | 4 +- 3 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 conf/whitelist.json create mode 100644 python/eggroll/backport/README.md diff --git a/conf/whitelist.json b/conf/whitelist.json new file mode 100644 index 000000000..cf42058ad --- /dev/null +++ b/conf/whitelist.json @@ -0,0 +1,138 @@ +{ + "builtins": [ + "int", + "list", + "set" + ], + "collections": [ + "defaultdict", + "OrderedDict" + ], + "eggroll.core.transfer_model": [ + "ErRollSiteHeader" + ], + "eggroll.roll_pair.task.storage": [ + "BSS" + ], + "federatedml.cipher_compressor.compressor": [ + "PackingCipherTensor", + "NormalCipherPackage", + "PackingCipherTensorPackage" + ], + "federatedml.ensemble.basic_algorithms.decision_tree.tree_core.feature_histogram": [ + "HistogramBag", + "FeatureHistogramWeights" + ], + "federatedml.ensemble.basic_algorithms.decision_tree.tree_core.feature_importance": [ + "FeatureImportance" + ], + "federatedml.ensemble.basic_algorithms.decision_tree.tree_core.g_h_optim": [ + "SplitInfoPackage", + "SplitInfoPackage2" + ], + "federatedml.ensemble.basic_algorithms.decision_tree.tree_core.node": [ + "Node" + ], + "federatedml.ensemble.basic_algorithms.decision_tree.tree_core.splitter": [ + "SplitInfo" + ], + "federatedml.evaluation.performance_recorder": [ + "PerformanceRecorder" + ], + "federatedml.feature.binning.bin_result": [ + "BinColResults" + ], + "federatedml.feature.binning.optimal_binning.bucket_info": [ + "Bucket" + ], + "federatedml.feature.binning.optimal_binning.heap": [ + "MinHeap", + "IvHeapNode", + "GiniHeapNode", + "ChiSquareHeapNode" + ], + "federatedml.feature.binning.quantile_summaries": [ + "SparseQuantileSummaries", + "Stats", + "QuantileSummaries" + ], + "federatedml.feature.fate_element_type": [ + "NoneType" + ], + "federatedml.feature.homo_feature_binning.homo_binning_base": [ + "SplitPointNode" + ], + "federatedml.feature.instance": [ + "Instance" + ], + "federatedml.feature.one_hot_encoder": [ + "TransferPair" + ], + "federatedml.feature.sparse_vector": [ + "SparseVector" + ], + "federatedml.framework.weights": [ + "NumpyWeights", + "TransferableWeights", + "NumericWeights", + "ListWeights", + "DictWeights", + "OrderDictWeights" + ], + "federatedml.linear_model.linear_model_weight": [ + "LinearModelWeights" + ], + "federatedml.secureprotol.fate_paillier": [ + "PaillierPublicKey", + "PaillierEncryptedNumber" + ], + "federatedml.secureprotol.fixedpoint": [ + "FixedPointNumber" + ], + "federatedml.secureprotol.number_theory.field.integers_modulo_prime_field": [ + "IntegersModuloPrimeElement" + ], + "federatedml.secureprotol.number_theory.group.twisted_edwards_curve_group": [ + "TwistedEdwardsCurveElement" + ], + "federatedml.secureprotol.symmetric_encryption.cryptor_executor": [ + "CryptoExecutor" + ], + "federatedml.secureprotol.symmetric_encryption.pohlig_hellman_encryption": [ + "PohligHellmanCiphertext", + "PohligHellmanCipherKey" + ], + "federatedml.statistic.intersect.intersect_preprocess": [ + "BitArray" + ], + "federatedml.statistic.statics": [ + "SummaryStatistics" + ], + "gmpy2": [ + "from_binary" + ], + "numpy": [ + "ndarray", + "dtype" + ], + "numpy.core.multiarray": [ + "scalar", + "_reconstruct" + ], + "numpy.core.numeric": [ + "_frombuffer" + ], + "tensorflow.python.framework.ops": [ + "convert_to_tensor" + ], + "torch._utils": [ + "_rebuild_tensor_v2" + ], + "ipcl_python.bindings.ipcl_bindings": [ + "ipclPublicKey" + ], + "ipcl_python.ipcl_python": [ + "PaillierPublicKey", + "PaillierEncryptedNumber" + ] +} diff --git a/python/eggroll/backport/README.md b/python/eggroll/backport/README.md new file mode 100644 index 000000000..528a235a9 --- /dev/null +++ b/python/eggroll/backport/README.md @@ -0,0 +1,5 @@ +## Backport of Eggroll-2.x + +This directory contains the backport api of Eggroll-2.x and may be used for FATE-1.x. +For users who use newer version api of Eggroll-3.x, +which is not compatible with FATE-2.x, this directory can be safely deleted. diff --git a/python/eggroll/backport/_serdes.py b/python/eggroll/backport/_serdes.py index 65c11b153..a408cb36f 100644 --- a/python/eggroll/backport/_serdes.py +++ b/python/eggroll/backport/_serdes.py @@ -14,6 +14,7 @@ # # +import io import importlib from pickle import Unpickler, UnpicklingError from pickle import dumps as p_dumps @@ -38,7 +39,8 @@ def serialize(obj) -> bytes: @staticmethod def deserialize(bytes) -> object: - return RestrictedUnpickler(bytes).load() + file = io.BytesIO(bytes) + return RestrictedUnpickler(file).load() class RestrictedUnpickler(Unpickler):