From 5b3c0b8c1d3b13362fb0055cd9e28dae36ea4a5f Mon Sep 17 00:00:00 2001 From: teo Date: Wed, 28 Jun 2023 20:03:27 +0300 Subject: [PATCH 01/11] init commit --- packages/syft/src/syft/client/client.py | 17 +++++++++++++++++ packages/syft/src/syft/util/env.py | 15 +++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 packages/syft/src/syft/util/env.py diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 63e79cae1c4..8550fd8db25 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -22,6 +22,7 @@ from requests.packages.urllib3.util.retry import Retry from tqdm import tqdm from typing_extensions import Self +import subprocess # relative from .. import __version__ @@ -52,6 +53,7 @@ from ..util.util import get_mb_size from ..util.util import thread_ident from ..util.util import verify_tls +from ..util.env import Env from .api import APIModule from .api import APIRegistry from .api import SignedSyftAPICall @@ -328,6 +330,21 @@ def __init__( self.post_init() + def get_env(self) -> Env: + res = subprocess.run("pip list | tail -n +3 | awk '{print $1,$2}'", + shell=True, check=True, executable='/bin/bash', + capture_output=True) + packages_dict = {} + for line in res.stdout.decode().split('\n')[:-1]: + print(line) + elems = line.split(" ") + package_name = elems[0] + version = elems[1] + if len(elems) == 3: + print(f"Warning, package {package_name} is installed in editable mode so errors might occur!!!") + packages_dict[package_name] = version + return Env(packages_dict=packages_dict) + def post_init(self) -> None: if self.metadata is None: self._fetch_node_metadata(self.credentials) diff --git a/packages/syft/src/syft/util/env.py b/packages/syft/src/syft/util/env.py new file mode 100644 index 00000000000..cc44998e898 --- /dev/null +++ b/packages/syft/src/syft/util/env.py @@ -0,0 +1,15 @@ +from ..types.syft_object import SyftObject, SYFT_OBJECT_VERSION_1 +from typing import Dict +import venv + +class Env(SyftObject): + __canonical_name__ = "Env" + __version__ = SYFT_OBJECT_VERSION_1 + packages_dict: Dict[str, str] + + @property + def packages(self): + return [(k, v) for k, v in self.packages_dict.items()] + + def create_local_env(self): + venv.EnvBuilder() \ No newline at end of file From efdf4610d4af5332241d92aa4612f8e38519f052 Mon Sep 17 00:00:00 2001 From: teo Date: Mon, 3 Jul 2023 12:26:15 +0300 Subject: [PATCH 02/11] changed pip list to freeze format --- packages/syft/src/syft/client/client.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 8550fd8db25..7e6db7d3be0 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -331,19 +331,20 @@ def __init__( self.post_init() def get_env(self) -> Env: - res = subprocess.run("pip list | tail -n +3 | awk '{print $1,$2}'", + res = subprocess.run("pip list --format=freeze", shell=True, check=True, executable='/bin/bash', capture_output=True) - packages_dict = {} - for line in res.stdout.decode().split('\n')[:-1]: - print(line) - elems = line.split(" ") - package_name = elems[0] - version = elems[1] - if len(elems) == 3: - print(f"Warning, package {package_name} is installed in editable mode so errors might occur!!!") - packages_dict[package_name] = version - return Env(packages_dict=packages_dict) + return res.stdout.decode() + # packages_dict = {} + # for line in res.stdout.decode().split('\n')[:-1]: + # print(line) + # elems = line.split(" ") + # package_name = elems[0] + # version = elems[1] + # if len(elems) == 3: + # print(f"Warning, package {package_name} is installed in editable mode so errors might occur!!!") + # packages_dict[package_name] = version + # return Env(packages_dict=packages_dict) def post_init(self) -> None: if self.metadata is None: From a1eb538024f43f697e01eaaf05e327675086f030 Mon Sep 17 00:00:00 2001 From: teo Date: Thu, 6 Jul 2023 17:12:37 +0300 Subject: [PATCH 03/11] fix lint --- packages/syft/src/syft/client/client.py | 14 +++++++++----- packages/syft/src/syft/util/env.py | 13 +++++++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 7e6db7d3be0..532a3172597 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -5,6 +5,7 @@ from enum import Enum import hashlib import json +import subprocess from typing import Any from typing import Dict from typing import Optional @@ -22,7 +23,6 @@ from requests.packages.urllib3.util.retry import Retry from tqdm import tqdm from typing_extensions import Self -import subprocess # relative from .. import __version__ @@ -47,13 +47,13 @@ from ..types.grid_url import GridURL from ..types.syft_object import SYFT_OBJECT_VERSION_1 from ..types.uid import UID +from ..util.env import Env from ..util.fonts import fonts_css from ..util.logger import debug from ..util.telemetry import instrument from ..util.util import get_mb_size from ..util.util import thread_ident from ..util.util import verify_tls -from ..util.env import Env from .api import APIModule from .api import APIRegistry from .api import SignedSyftAPICall @@ -331,9 +331,13 @@ def __init__( self.post_init() def get_env(self) -> Env: - res = subprocess.run("pip list --format=freeze", - shell=True, check=True, executable='/bin/bash', - capture_output=True) + res = subprocess.run( + "pip list --format=freeze", + shell=True, + check=True, + executable="/bin/bash", + capture_output=True, + ) return res.stdout.decode() # packages_dict = {} # for line in res.stdout.decode().split('\n')[:-1]: diff --git a/packages/syft/src/syft/util/env.py b/packages/syft/src/syft/util/env.py index cc44998e898..e84b5118494 100644 --- a/packages/syft/src/syft/util/env.py +++ b/packages/syft/src/syft/util/env.py @@ -1,15 +1,20 @@ -from ..types.syft_object import SyftObject, SYFT_OBJECT_VERSION_1 +# stdlib from typing import Dict import venv +# relative +from ..types.syft_object import SYFT_OBJECT_VERSION_1 +from ..types.syft_object import SyftObject + + class Env(SyftObject): __canonical_name__ = "Env" __version__ = SYFT_OBJECT_VERSION_1 packages_dict: Dict[str, str] - + @property def packages(self): return [(k, v) for k, v in self.packages_dict.items()] - + def create_local_env(self): - venv.EnvBuilder() \ No newline at end of file + venv.EnvBuilder() From b0b4c13fbffcbdf1d1550ed958abbcb83f74072c Mon Sep 17 00:00:00 2001 From: teo Date: Thu, 13 Jul 2023 17:27:11 +0300 Subject: [PATCH 04/11] added get_env method --- packages/syft/src/syft/client/client.py | 9 +------ packages/syft/src/syft/node/node.py | 3 +++ .../syft/service/metadata/metadata_service.py | 27 +++++++++++++++++++ 3 files changed, 31 insertions(+), 8 deletions(-) create mode 100644 packages/syft/src/syft/service/metadata/metadata_service.py diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 0afee859b55..22e7c4f588b 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -358,14 +358,7 @@ def __init__( self.post_init() def get_env(self) -> str: - res = subprocess.run( - "pip list --format=freeze", - shell=True, - check=True, - executable="/bin/bash", - capture_output=True, - ) - return res.stdout.decode() + return self.api.services.metadata.get_env() def post_init(self) -> None: if self.metadata is None: diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 637c5286343..53d4108d1d0 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -73,6 +73,7 @@ from ..service.user.user import User from ..service.user.user import UserCreate from ..service.user.user_roles import ServiceRole +from ..service.metadata.metadata_service import MetadataService from ..service.user.user_service import UserService from ..service.user.user_stash import UserStash from ..store.dict_document_store import DictStoreConfig @@ -211,6 +212,7 @@ def __init__( DataSubjectMemberService, ProjectService, EnclaveService, + MetadataService, ] if services is None else services @@ -456,6 +458,7 @@ def _construct_services(self): DataSubjectMemberService, ProjectService, EnclaveService, + MetadataService ] if OBLV: diff --git a/packages/syft/src/syft/service/metadata/metadata_service.py b/packages/syft/src/syft/service/metadata/metadata_service.py new file mode 100644 index 00000000000..62676cb9a15 --- /dev/null +++ b/packages/syft/src/syft/service/metadata/metadata_service.py @@ -0,0 +1,27 @@ +import gevent.subprocess as subprocess +from ...serde.serializable import serializable +from ...util.telemetry import instrument +from ..service import AbstractService +from ..service import service_method +from ..context import AuthedServiceContext +from ..user.user_roles import GUEST_ROLE_LEVEL +from ..response import SyftError +from ..response import SyftSuccess +from ...store.document_store import DocumentStore + +@instrument +@serializable() +class MetadataService(AbstractService): + + def __init__(self, store: DocumentStore) -> None: + self.store = store + + @service_method(path="metadata.get_env", name="get_env", roles=GUEST_ROLE_LEVEL) + def get_env(self, context: AuthedServiceContext): + + res = subprocess.getoutput( + "pip list --format=freeze", + ) + import sys + print(res, file=sys.stderr) + return SyftSuccess(message=res.stdout.decode()) \ No newline at end of file From 2c88fd4fc9bd0f79b69680894b4db9e45e160d83 Mon Sep 17 00:00:00 2001 From: teo Date: Thu, 13 Jul 2023 18:51:17 +0300 Subject: [PATCH 05/11] modified request repr --- packages/syft/src/syft/client/client.py | 1 - .../syft/src/syft/client/gateway_client.py | 4 ++- packages/syft/src/syft/node/node.py | 4 +-- .../syft/service/metadata/metadata_service.py | 31 ++++++++++++++----- .../syft/src/syft/service/request/request.py | 8 +++-- .../src/syft/service/user/user_service.py | 9 ++++++ .../syft/src/syft/service/user/user_stash.py | 5 +++ 7 files changed, 48 insertions(+), 14 deletions(-) diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index cea2684ddee..7f4accfe75a 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -5,7 +5,6 @@ from enum import Enum import hashlib import json -import subprocess from typing import Any from typing import Callable from typing import Dict diff --git a/packages/syft/src/syft/client/gateway_client.py b/packages/syft/src/syft/client/gateway_client.py index a7806be0fe6..eaa1c2a1ad3 100644 --- a/packages/syft/src/syft/client/gateway_client.py +++ b/packages/syft/src/syft/client/gateway_client.py @@ -64,7 +64,9 @@ def proxy_client_for( def domains(self) -> Optional[Union[List[NodePeer], SyftError]]: if not self.api.has_service("network"): return None - return self.api.services.network.get_peers_by_type(node_type=NodeType.DOMAIN) + domains = self.api.services.network.get_peers_by_type(node_type=NodeType.DOMAIN) + domains.append(self) + return domains @property def enclaves(self) -> Optional[Union[List[NodePeer], SyftError]]: diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 8973a888b64..993519eb055 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -52,6 +52,7 @@ from ..service.data_subject.data_subject_service import DataSubjectService from ..service.dataset.dataset_service import DatasetService from ..service.enclave.enclave_service import EnclaveService +from ..service.metadata.metadata_service import MetadataService from ..service.metadata.node_metadata import NodeMetadata from ..service.network.network_service import NetworkService from ..service.notification.notification_service import NotificationService @@ -74,7 +75,6 @@ from ..service.user.user import User from ..service.user.user import UserCreate from ..service.user.user_roles import ServiceRole -from ..service.metadata.metadata_service import MetadataService from ..service.user.user_service import UserService from ..service.user.user_stash import UserStash from ..store.dict_document_store import DictStoreConfig @@ -459,7 +459,7 @@ def _construct_services(self): DataSubjectMemberService, ProjectService, EnclaveService, - MetadataService + MetadataService, ] if OBLV: diff --git a/packages/syft/src/syft/service/metadata/metadata_service.py b/packages/syft/src/syft/service/metadata/metadata_service.py index 62676cb9a15..7b859470cd2 100644 --- a/packages/syft/src/syft/service/metadata/metadata_service.py +++ b/packages/syft/src/syft/service/metadata/metadata_service.py @@ -1,27 +1,42 @@ +# third party import gevent.subprocess as subprocess + +# relative from ...serde.serializable import serializable +from ...store.document_store import DocumentStore from ...util.telemetry import instrument +from ..context import AuthedServiceContext +from ..response import SyftSuccess from ..service import AbstractService from ..service import service_method -from ..context import AuthedServiceContext from ..user.user_roles import GUEST_ROLE_LEVEL -from ..response import SyftError -from ..response import SyftSuccess -from ...store.document_store import DocumentStore + @instrument @serializable() class MetadataService(AbstractService): - def __init__(self, store: DocumentStore) -> None: self.store = store - + + @service_method( + path="metadata.get_metadata", name="get_metadata", roles=GUEST_ROLE_LEVEL + ) + def get_metadata(self, context: AuthedServiceContext): + return context.node.metadata + + # @service_method(path="metadata.get_admin", name="get_admin", roles=GUEST_ROLE_LEVEL) + # def get_admin(self, context: AuthedServiceContext): + # user_service = context.node.get_service("userservice") + # admin_user = user_service.get_all(context=context)[0] + # return admin_user + @service_method(path="metadata.get_env", name="get_env", roles=GUEST_ROLE_LEVEL) def get_env(self, context: AuthedServiceContext): - res = subprocess.getoutput( "pip list --format=freeze", ) + # stdlib import sys + print(res, file=sys.stderr) - return SyftSuccess(message=res.stdout.decode()) \ No newline at end of file + return SyftSuccess(message=res.stdout.decode()) diff --git a/packages/syft/src/syft/service/request/request.py b/packages/syft/src/syft/service/request/request.py index bf634075647..2a73c6b38b7 100644 --- a/packages/syft/src/syft/service/request/request.py +++ b/packages/syft/src/syft/service/request/request.py @@ -189,7 +189,8 @@ def _repr_html_(self) -> Any: self.node_uid, self.syft_client_verify_key, ) - + metadata = api.services.metadata.get_metadata() + admin_user = api.services.user.get_admin() return f"""