quickstart| \n", - " š quickstart\n", - " | \n", - "\n", - " š§āāļø Install Wizard\n", - " | \n", - "
Step 1. Run quickstart |
| Step 2. Download a Tutorial |
| \n", - " š quickstart / 01-install-wizard.ipynb\n", - " | \n", - "
| Step 1. Import Install š§š½āāļø Wizard BETA |
| Step 2. HAGrid Updates |
| Step 3. Installing PySyft |
| Step 4. Python Server |
| Step 5. Docker Setup (Optional) |
| Step 6. Start a Test Domain |
| Step 7. Check Domain Health |
| Step 8. Domain Login |
| Step 9. Shutdown Domain |
| ā Install š§š½āāļø Wizard Complete |
Command:\n "
- + f"[ ]!{issue.command}"
- )
- output += "\n"
-
- return NBOutput(output).to_html()
- else:
- if len(issues) > 0:
- output += "šØ Some issues were found\n"
- for issue in issues:
- output += f"\nIssue: {issue.description}\n"
- if issue.solution != "":
- output += f"\nSolution:\n{issue.solution}\n"
- if issue.command != "":
- output += "\nCommand:\n" + f"{issue.command} "
- output += "\n"
-
- if len(output) > 0:
- print(output)
- return None # type: ignore
-
-
-def check_grid_docker(
- display: bool = True, output_in_text: bool = False
-) -> dict[str, Dependency] | NBOutput:
- try:
- deps: dict[str, Dependency] = {}
- deps["git"] = DependencyGridGit(name="git")
- deps["docker"] = DependencyGridDocker(name="docker")
- deps["docker_compose"] = DependencyGridDockerCompose(name="docker compose")
- return check_deps(
- of="Grid", deps=deps, display=display, output_in_text=output_in_text
- )
- except Exception as e:
- try:
- if display and not output_in_text:
- return NBOutput(debug_exception(e=e)).to_html()
- except Exception: # nosec
- pass
-
- print(e)
- raise e
-
-
-def debug_exception(e: Exception) -> str:
- exception = (
- f'" + json.dumps(gather_debug(), indent=4, sort_keys=True) + ""
- )
- exception += "\n"
- exception += traceback.format_exc()
- exception += (
- "\n=================================================================\n\n"
- )
- return exception
-
-
-def check_syft_deps(display: bool = True) -> dict[str, Dependency] | NBOutput:
- try:
- deps: dict[str, Dependency] = {}
- deps["os"] = DependencySyftOS(name="os")
- deps["python"] = DependencySyftPython(name="python")
- return check_deps(of="Syft", deps=deps, display=display)
- except Exception as e:
- try:
- if display:
- return NBOutput(debug_exception(e=e)).to_html()
- except Exception: # nosec
- pass
-
- print(e)
- raise e
-
-
-def check_hagrid(display: bool = True) -> dict[str, Dependency] | NBOutput:
- try:
- deps: dict[str, Dependency] = {}
- deps["hagrid"] = DependencyPyPI(
- package_name="hagrid",
- package_display_name="HAGrid",
- update_available_issue=hagrid_update_available,
- )
- return check_deps(deps=deps, display=display)
- except Exception as e:
- try:
- if display:
- return NBOutput(debug_exception(e=e)).to_html()
- except Exception: # nosec
- pass
-
- print(e)
- raise e
-
-
-def check_syft(
- display: bool = True, pre: bool = False
-) -> dict[str, Dependency] | NBOutput:
- try:
- deps: dict[str, Dependency] = {}
- deps["os"] = DependencySyftOS(name="os")
- deps["python"] = DependencySyftPython(name="python")
- deps["syft"] = DependencyPyPI(
- package_name="syft",
- package_display_name="Syft",
- pre=pre,
- install_issue=syft_install,
- update_available_issue=syft_update_available,
- )
- return check_deps(deps=deps, display=display)
- except Exception as e:
- try:
- if display:
- return NBOutput(debug_exception(e=e)).to_html()
- except Exception: # nosec
- pass
-
- print(e)
- raise e
-
-
-PACKAGE_MANAGER_COMMANDS = {
- "git": {
- "macos": "brew install git",
- "windows": 'choco install git.install --params "/GitAndUnixToolsOnPath /WindowsTerminal /NoAutoCrlf" -y',
- "linux": "sudo apt update && sudo apt install git",
- "backup_url": "https://git-scm.com/downloads",
- },
- "docker": {
- "macos": "brew install --cask docker",
- "windows": "choco install docker-desktop -y",
- "linux": "curl -fsSL https://get.docker.com -o get-docker.sh && chmod +777 get-docker.sh && ./get-docker.sh",
- "backup_url": "https://www.docker.com/products/docker-desktop/",
- },
- "docker_compose": {
- "macos": "brew install --cask docker",
- "windows": "choco install docker-desktop -y",
- "linux": (
- "mkdir -p ~/.docker/cli-plugins\n"
- + "DOCKER_COMPOSE_VERSION=v2.21.0\n"
- + "curl -sSL https://github.com/docker/compose/releases/download/"
- + "${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 "
- + "-o ~/.docker/cli-plugins/docker-compose\n"
- + "chmod +x ~/.docker/cli-plugins/docker-compose"
- ),
- "backup_url": "https://github.com/docker/compose",
- },
-}
-
-PACKAGE_MANAGERS = {
- "macos": "brew",
- "windows": "choco",
- "linux": "apt",
-}
-
-
-def os_package_manager_install_cmd(
- package_name: str, package_display_name: str, output_in_text: bool = False
-) -> tuple[str | None, str | None]:
- os = ENVIRONMENT["os"].lower()
- cmd = None
- url = None
- package_manager = PACKAGE_MANAGERS[os]
- if (
- package_name in PACKAGE_MANAGER_COMMANDS
- and os in PACKAGE_MANAGER_COMMANDS[package_name]
- ):
- cmd = PACKAGE_MANAGER_COMMANDS[package_name][os]
- if (
- package_name in PACKAGE_MANAGER_COMMANDS
- and "backup_url" in PACKAGE_MANAGER_COMMANDS[package_name]
- ):
- url = PACKAGE_MANAGER_COMMANDS[package_name]["backup_url"]
-
- solution = ""
-
- if not output_in_text:
- if cmd:
- solution += f"- You can install {package_display_name} with {package_manager}\n"
- if url:
- if cmd:
- solution += "- Alternatively, you "
- else:
- solution += "- You "
- solution += f"can download and install {package_display_name}"
- solution += f'from {url}'
- else:
- if cmd:
- solution += (
- f"- You can install {package_display_name} with {package_manager}\n"
- )
- if url:
- if cmd:
- solution += "- Alternatively, you "
- else:
- solution += "- You "
- solution += f"can download and install {package_display_name} from {url}"
-
- return (cmd, solution)
-
-
-def docker_compose_install() -> SetupIssue:
- command, solution = os_package_manager_install_cmd(
- package_name="docker_compose", package_display_name="Docker Compose"
- )
- return SetupIssue(
- issue_name="docker_compose_install",
- description="You do not have Docker Compose v2 installed.",
- command=command,
- solution=solution,
- )
-
-
-def docker_install() -> SetupIssue:
- command, solution = os_package_manager_install_cmd(
- package_name="docker", package_display_name="Docker"
- )
- return SetupIssue(
- issue_name="docker_install",
- description="You do not have Docker installed.",
- command=command,
- solution=solution,
- )
-
-
-def git_install(output_in_text: bool = False) -> SetupIssue:
- command, solution = os_package_manager_install_cmd(
- package_name="git", package_display_name="Git", output_in_text=output_in_text
- )
- return SetupIssue(
- issue_name="git_install",
- description="You do not have Git installed.",
- command=command,
- solution=solution,
- )
-
-
-def syft_install(pre: bool = False) -> SetupIssue:
- command = "pip install -U syft"
- if pre:
- # command += " --pre"
- pass
- return SetupIssue(
- issue_name="syft_install",
- description="You have not installed Syft.",
- command=command,
- solution="You can install Syft with pip.",
- )
-
-
-def syft_update_available(current_version: Version, new_version: Version) -> SetupIssue:
- return SetupIssue(
- issue_name="syft_update_available",
- description=(
- "A new release of Syft is available: "
- + f"{str(current_version)} -> {str(new_version)}."
- ),
- command=f"pip install syft=={new_version}",
- solution="You can upgrade Syft with pip.",
- )
-
-
-def hagrid_update_available(
- current_version: Version, new_version: Version
-) -> SetupIssue:
- return SetupIssue(
- issue_name="hagrid_update_available",
- description=(
- "A new release of HAGrid is available: "
- + f"{str(current_version)} -> {str(new_version)}."
- ),
- command=f"pip install -U hagrid=={new_version}",
- solution="You can upgrade HAGrid with pip.",
- )
-
-
-def python_version_unsupported() -> SetupIssue:
- return SetupIssue(
- issue_name="python_version_unsupported",
- description=(
- f"Syft supports Python >= {SYFT_MINIMUM_PYTHON_VERSION_STRING} "
- + f"and <= {SYFT_MAXIMUM_PYTHON_VERSION_STRING}"
- ),
- command="",
- solution="You must install a compatible version of Python",
- )
diff --git a/packages/hagrid/hagrid/dummynum.py b/packages/hagrid/hagrid/dummynum.py
deleted file mode 100644
index 06b28f28682..00000000000
--- a/packages/hagrid/hagrid/dummynum.py
+++ /dev/null
@@ -1,28 +0,0 @@
-# stdlib
-from typing import Any
-
-# a dummy enum
-
-
-class Meta(type):
- # any property returns another dummy which can also be executed
- def __getattribute__(cls, name: str) -> Any:
- try:
- return super().__getattribute__(name)
- except Exception: # nosec
- pass
- return return_dummy()
-
-
-# this lets us prevent runtime errors of missing types in older syft
-class DummyNum(metaclass=Meta):
- def __init__(self, *args: Any, **kwargs: Any) -> None:
- pass
-
- def __call__(self, *args: Any, **kwargs: Any) -> Any:
- return self
-
-
-def return_dummy() -> DummyNum:
- # this lets us create the sub class in the parent meta on getattr
- return DummyNum()
diff --git a/packages/hagrid/hagrid/exceptions.py b/packages/hagrid/hagrid/exceptions.py
deleted file mode 100644
index 0bb68a0dd55..00000000000
--- a/packages/hagrid/hagrid/exceptions.py
+++ /dev/null
@@ -1,2 +0,0 @@
-class MissingDependency(Exception):
- pass
diff --git a/packages/hagrid/hagrid/file.py b/packages/hagrid/hagrid/file.py
deleted file mode 100644
index 378f5fdcf10..00000000000
--- a/packages/hagrid/hagrid/file.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# stdlib
-import os
-
-
-def user_hagrid_profile() -> str:
- dir_path = os.path.expanduser("~/.hagrid")
- os.makedirs(dir_path, exist_ok=True)
- return os.path.abspath(dir_path)
diff --git a/packages/hagrid/hagrid/git_check.py b/packages/hagrid/hagrid/git_check.py
deleted file mode 100644
index c98028f0c52..00000000000
--- a/packages/hagrid/hagrid/git_check.py
+++ /dev/null
@@ -1,15 +0,0 @@
-# relative
-from .deps import DependencyGridGit
-from .deps import check_deps
-
-
-def verify_git_installation() -> None:
- dep = DependencyGridGit(name="git", output_in_text=True)
- deps = {}
- deps["git"] = dep
- check_deps(of="Git", deps=deps, display=False, output_in_text=True) # type: ignore
- if dep.issues:
- exit(1)
-
-
-verify_git_installation()
diff --git a/packages/hagrid/hagrid/grammar.py b/packages/hagrid/hagrid/grammar.py
deleted file mode 100644
index 62f98d47fe8..00000000000
--- a/packages/hagrid/hagrid/grammar.py
+++ /dev/null
@@ -1,365 +0,0 @@
-# future
-from __future__ import annotations
-
-# stdlib
-from collections.abc import Callable
-import socket
-from typing import Any
-
-# relative
-from .deps import allowed_hosts
-from .lib import find_available_port
-
-ALLOWED_NODE_TYPES = ["domain", "network", "gateway", "enclave"]
-
-
-class BadGrammar(Exception):
- pass
-
-
-class GrammarVerb:
- def __init__(
- self,
- command: str,
- full_sentence: list[dict[str, Any]],
- abbreviations: dict[int, list[str | None]],
- ) -> None:
- self.grammar: list[GrammarTerm | HostGrammarTerm | SourceGrammarTerm] = []
- self.command = command
- self.full_sentence = full_sentence
- self.abbreviations = abbreviations
-
- def get_named_term_grammar(self, name: str) -> GrammarTerm:
- for term in self.grammar:
- if term.name == name and isinstance(term, GrammarTerm):
- return term
- raise BadGrammar(f"GrammarTerm with {name} not found in {self.grammar}")
-
- def get_named_term_hostgrammar(self, name: str) -> HostGrammarTerm:
- for term in self.grammar:
- if term.name == name and isinstance(term, HostGrammarTerm):
- return term
- raise BadGrammar(f"HostGrammarTerm with {name} not found in {self.grammar}")
-
- def get_named_term_type(
- self, name: str, term_type: str | None = None
- ) -> GrammarTerm | HostGrammarTerm:
- if term_type == "host":
- return self.get_named_term_hostgrammar(name=name)
- return self.get_named_term_grammar(name=name)
-
- def set_named_term_type(
- self, name: str, new_term: GrammarTerm, term_type: str | None = None
- ) -> None:
- new_grammar = []
- for term in self.grammar:
- found = False
- if term.name == name:
- if term_type is not None and term.type == term_type:
- found = True
- elif term_type is None:
- found = True
- if not found:
- new_grammar.append(term)
- else:
- new_grammar.append(new_term)
- self.grammar = new_grammar
-
- def load_grammar(
- self, grammar: list[GrammarTerm | HostGrammarTerm | SourceGrammarTerm]
- ) -> None:
- self.grammar = grammar
-
-
-class GrammarTerm:
- def __init__(
- self,
- type: str,
- name: str,
- default: str | Callable | None = None,
- options: list | None = None,
- example: str | None = None,
- **kwargs: Any,
- ) -> None:
- self.raw_input: str | None = None
- self.input: str | None = None
- self.type = type
- self.name = name
- self.default = default
- self.options = options if options is not None else []
- self.example = example
-
- @property
- def snake_input(self) -> str | None:
- if self.input:
- return self.input.lower().replace(" ", "_")
- return None
-
- @property
- def kebab_input(self) -> str | None:
- if self.input:
- return self.input.lower().replace(" ", "-")
- return None
-
- def __repr__(self) -> str:
- return f"<{type(self).__name__}: {self.name}<{self.type}>: {self.input} [raw: {self.raw_input}]>"
-
- def get_example(self) -> str:
- return_value = self.example if self.example else self.default
- if callable(return_value):
- return_value = return_value()
- return str(return_value)
-
- # no op
- def custom_parsing(self, input: str) -> str:
- return input
-
- def parse_input(self, input: str | None) -> None:
- self.raw_input = input
- if input is None and self.default is None:
- raise BadGrammar(
- f"{self.name} has no default, please use one of the following options: {self.options}"
- )
- if input is None:
- if isinstance(self.default, str):
- input = self.default
- elif callable(self.default):
- input = self.default()
-
- if len(self.options) > 0 and input not in self.options:
- raise BadGrammar(
- f"{input} is not valid for {self.name} please use one of the following options: {self.options}"
- )
-
- self.input = self.custom_parsing(input=input) if input else input
-
-
-class HostGrammarTerm(GrammarTerm):
- @property
- def host(self) -> str | None:
- return self.parts()[0]
-
- @property
- def port(self) -> int | None:
- return self.parts()[1]
-
- @property
- def search(self) -> bool:
- return bool(self.parts()[2])
-
- @property
- def port_tls(self) -> int:
- if self.port == 80:
- return 443
- return 444
-
- @property
- def free_port(self) -> int:
- if self.port is None:
- raise BadGrammar(
- f"{type(self)} unable to check if port {self.port} is free"
- )
- return find_available_port(host="localhost", port=self.port, search=self.search)
-
- @property
- def free_port_tls(self) -> int:
- if self.port_tls is None:
- raise BadGrammar(
- f"{type(self)} unable to check if tls port {self.port_tls} is free"
- )
- return find_available_port(host="localhost", port=self.port_tls, search=True)
-
- def parts(self) -> tuple[str | None, int | None, bool]:
- host = None
- port: int | None = None
- search = False
- if self.input:
- parts = self.input.split(":")
- host = parts[0]
- if len(parts) > 1:
- port_str = parts[1]
- if port_str.endswith("+"):
- search = True
- port_str = port_str[0:-1]
- port = int(port_str)
- return (host, port, search)
-
- def validate_host(self, host_or_ip: str) -> bool:
- try:
- if socket.gethostbyname(host_or_ip) == host_or_ip:
- return True
- elif socket.gethostbyname(host_or_ip) != host_or_ip:
- return True
- except socket.gaierror:
- raise BadGrammar(
- f"{host_or_ip} is not valid for {self.name}. Try an IP, hostname or docker, vm, aws, azure or gcp"
- )
- return False
-
- def validate_port(self, port: str) -> bool:
- try:
- if port.endswith("+"):
- int(port[0:-1])
- else:
- int(port)
- except Exception: # nosec
- raise BadGrammar(
- f"{port} is not a valid port option. Try: {self.get_example()}"
- )
- return True
-
- def custom_parsing(self, input: str) -> str:
- colons = input.count(":")
- host = input
- port = None
- if colons > 1:
- raise BadGrammar(
- f"You cannot have more than one : for {self.name}, try: {self.get_example()}"
- )
- elif colons == 1:
- parts = input.split(":")
- host = parts[0]
- port = parts[1]
-
- if port is None:
- if host == "docker":
- port = "8081+" # default
- else:
- port = "80" # default
-
- if host not in allowed_hosts:
- _ = self.validate_host(host_or_ip=host)
-
- _ = self.validate_port(port=port)
-
- return f"{host}:{port}"
-
-
-class SourceGrammarTerm(GrammarTerm):
- def custom_parsing(self, input: str) -> str:
- trimmed = input
- if trimmed.startswith("http://"):
- trimmed = trimmed.replace("http://", "")
- if trimmed.startswith("https://"):
- trimmed = trimmed.replace("https://", "")
- if trimmed.startswith("github.com/"):
- trimmed = trimmed.replace("github.com/", "")
-
- parts = trimmed.split("/")
- if "tree" not in input or len(parts) < 4:
- raise BadGrammar(
- f"{self.name} should be a valid github.com repo branch url. Try: {self.get_example()}"
- )
-
- repo = f"{parts[0]}/{parts[1]}"
- branch = "/".join(parts[3:])
-
- return f"{repo}:{branch}"
-
-
-def validate_arg_count(arg_count: int, verb: GrammarVerb) -> bool:
- valid = True
-
- if arg_count not in verb.abbreviations:
- error_str = f"Command {verb.command} supports the following invocations:\n"
- for count in sorted(verb.abbreviations.keys()):
- abbreviation = verb.abbreviations[count]
- example_terms = []
- for i, term_type in enumerate(abbreviation):
- if term_type is not None:
- term_settings = verb.full_sentence[i]
- example = term_settings["klass"](**term_settings).get_example()
- example_terms.append(example)
- error_str += f"{count} args: {verb.command} {' '.join(example_terms)}\n"
-
- raise BadGrammar(error_str)
-
- return valid
-
-
-def launch_shorthand_support(args: tuple) -> tuple:
- """When launching, we want to be able to default to 'domain' if it's not provided, to launch
- nodes when no name is provided, and to support node names which have multiple words.
-
- hagrid launch -> hagrid launch domain
- hagrid launch United Nations -> hagrid launch "United Nations" domain
- hagrid launch United Nations domain -> hagrid launch "United Nations" domain
- hagrid launch on docker -> hagrid launch domain on docker
-
- """
-
- # Some mild analysis
- found_node_type = False
- preposition_position = 10000
- for i, arg in enumerate(args):
- if arg in ALLOWED_NODE_TYPES:
- found_node_type = True
-
- if arg.strip() in ["to", "from"]:
- if i < preposition_position:
- preposition_position = i
-
- _args = list(args)
-
- # Default to domain if it's not provided
- if not found_node_type:
- if preposition_position != 10000:
- _args.insert(preposition_position, "domain")
- preposition_position += 1
- else:
- _args = _args + ["domain"]
-
- # if there are no prepositions and the domain/network is the last word
- if preposition_position == 10000 and _args[-1] in ALLOWED_NODE_TYPES:
- _args = [" ".join(_args[:-1])] + _args[-1:]
-
- # if there are prepositions then combine the words in the name if there are multiple
- elif preposition_position != 10000:
- _args = [" ".join(_args[: preposition_position - 1])] + _args[
- preposition_position - 1 :
- ]
-
- # if there wasn't a name provided - make sure we don't have an empty place in the list
- # so that later logic will generate a name
- if _args[0] == "":
- _args = _args[1:]
-
- args = tuple(_args)
-
- return args
-
-
-def parse_grammar(args: tuple, verb: GrammarVerb) -> list[GrammarTerm]:
- # if the command is a launch, check if any shorthands were employed
- if verb.command == "launch":
- args = launch_shorthand_support(args=args)
-
- arg_list = list(args)
- arg_count = len(arg_list)
- errors = []
- if validate_arg_count(arg_count=arg_count, verb=verb):
- terms = []
- abbreviation = verb.abbreviations[arg_count]
- for i, term_type in enumerate(abbreviation):
- if term_type is None:
- arg = None # use None so we get the default
- else:
- arg = arg_list.pop(0) # use a real arg
-
- term_settings = verb.full_sentence[i]
-
- try:
- term = term_settings["klass"](**term_settings)
- term.parse_input(arg)
- terms.append(term)
-
- except BadGrammar as e:
- errors.append(str(e))
-
- if len(errors) > 0:
- raise BadGrammar("\n".join(errors))
-
- # make command
- return terms
- else:
- raise BadGrammar("Grammar is not valid")
diff --git a/packages/hagrid/hagrid/img/hagrid.png b/packages/hagrid/hagrid/img/hagrid.png
deleted file mode 100644
index 2b4dbd75d41..00000000000
Binary files a/packages/hagrid/hagrid/img/hagrid.png and /dev/null differ
diff --git a/packages/hagrid/hagrid/img/hagrid2.png b/packages/hagrid/hagrid/img/hagrid2.png
deleted file mode 100644
index 8df9d58d147..00000000000
Binary files a/packages/hagrid/hagrid/img/hagrid2.png and /dev/null differ
diff --git a/packages/hagrid/hagrid/land.py b/packages/hagrid/hagrid/land.py
deleted file mode 100644
index 1c138c1971b..00000000000
--- a/packages/hagrid/hagrid/land.py
+++ /dev/null
@@ -1,60 +0,0 @@
-# stdlib
-
-# relative
-from .grammar import GrammarTerm
-from .grammar import GrammarVerb
-from .grammar import HostGrammarTerm
-
-
-def get_land_verb() -> GrammarVerb:
- full_sentence = [
- {
- "name": "node_name",
- "type": "adjective",
- "klass": GrammarTerm,
- "example": "'my_domain'",
- },
- {
- "name": "preposition",
- "type": "preposition",
- "klass": GrammarTerm,
- "default": "at",
- "options": ["at", "on"],
- },
- {
- "name": "host",
- "type": "propernoun",
- "klass": HostGrammarTerm,
- "default": "docker",
- "example": "docker",
- },
- ]
-
- abbreviations: dict[int, list[str | None]] = {
- 3: [
- "adjective",
- "preposition",
- "propernoun",
- ], # node_name # at # host
- 2: [
- "adjective",
- None,
- "propernoun",
- ], # node_name # ignore # host
- 1: [
- "adjective",
- None,
- None,
- ], # node_name # ignore # ignore
- 0: [
- None,
- None,
- None,
- ], # ignore # ignore # ignore
- }
-
- return GrammarVerb(
- command="land",
- full_sentence=full_sentence,
- abbreviations=abbreviations,
- )
diff --git a/packages/hagrid/hagrid/launch.py b/packages/hagrid/hagrid/launch.py
deleted file mode 100644
index c6cc785da50..00000000000
--- a/packages/hagrid/hagrid/launch.py
+++ /dev/null
@@ -1,113 +0,0 @@
-# stdlib
-
-# relative
-from .cache import DEFAULT_BRANCH
-from .grammar import ALLOWED_NODE_TYPES
-from .grammar import GrammarTerm
-from .grammar import GrammarVerb
-from .grammar import HostGrammarTerm
-from .grammar import SourceGrammarTerm
-from .names import random_name
-
-
-def get_launch_verb() -> GrammarVerb:
- full_sentence = [
- {
- "name": "node_name",
- "type": "propernoun",
- "klass": GrammarTerm,
- "default": random_name,
- "example": "'my_domain'",
- },
- {
- "name": "node_type",
- "type": "object",
- "klass": GrammarTerm,
- "default": "domain",
- "options": ALLOWED_NODE_TYPES,
- },
- {
- "name": "preposition",
- "type": "preposition",
- "klass": GrammarTerm,
- "default": "to",
- "options": ["to"],
- },
- {
- "name": "host",
- "type": "propernoun",
- "klass": HostGrammarTerm,
- "default": "docker",
- "example": "docker:8081+",
- },
- {
- "name": "preposition",
- "type": "preposition",
- "klass": GrammarTerm,
- "default": "from",
- "options": ["from"],
- },
- {
- "name": "source",
- "type": "propernoun",
- "klass": SourceGrammarTerm,
- "default": f"github.com/OpenMined/PySyft/tree/{DEFAULT_BRANCH}",
- },
- ]
-
- abbreviations: dict[int, list[str | None]] = {
- 6: [
- "propernoun", # name
- "object", # node_type
- "preposition", # to
- "propernoun", # host
- "preposition", # from
- "propernoun", # source
- ],
- 5: [
- None, # name
- "object", # node_type
- "preposition", # to
- "propernoun", # host
- "preposition", # from
- "propernoun", # source
- ],
- 4: [
- "propernoun", # name
- "object", # node_type
- "preposition", # to
- "propernoun", # host
- None, # ignore
- None, # ignore
- ],
- 3: [
- None, # ignore
- "object", # node_type
- "preposition", # to
- "propernoun", # host
- None, # ignore
- None, # ignore
- ],
- 2: [
- "propernoun", # name
- "object", # node_type
- None, # ignore
- None, # ignore
- None, # ignore
- None, # ignore
- ],
- 1: [
- None, # ignore
- "object", # node_type
- None, # ignore
- None, # ignore
- None, # ignore
- None, # ignore
- ],
- }
-
- return GrammarVerb(
- command="launch",
- full_sentence=full_sentence,
- abbreviations=abbreviations,
- )
diff --git a/packages/hagrid/hagrid/lib.py b/packages/hagrid/hagrid/lib.py
deleted file mode 100644
index 057f77160f7..00000000000
--- a/packages/hagrid/hagrid/lib.py
+++ /dev/null
@@ -1,472 +0,0 @@
-# stdlib
-from enum import Enum
-import hashlib
-import importlib
-import importlib.machinery
-import importlib.util
-import json
-import os
-from pathlib import Path
-import random
-import shutil
-import socket
-import subprocess # nosec
-
-# third party
-import git
-import requests
-import rich
-from rich import console
-from rich import progress
-from rich.table import Table
-
-# relative
-from .cache import DEFAULT_BRANCH
-from .mode import EDITABLE_MODE
-from .mode import hagrid_root
-
-
-class GitRemoteProgress(git.RemoteProgress):
- # CREDITS: https://splunktool.com/python-progress-bar-for-git-clone
- OP_CODES = [
- "BEGIN",
- "CHECKING_OUT",
- "COMPRESSING",
- "COUNTING",
- "END",
- "FINDING_SOURCES",
- "RECEIVING",
- "RESOLVING",
- "WRITING",
- ]
- OP_CODE_MAP = {
- getattr(git.RemoteProgress, _op_code): _op_code for _op_code in OP_CODES
- }
-
- def __init__(self) -> None:
- super().__init__()
- self.progressbar = progress.Progress(
- progress.SpinnerColumn(),
- # *progress.Progress.get_default_columns(),
- progress.TextColumn("[progress.description]{task.description}"),
- progress.BarColumn(),
- progress.TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
- "eta",
- progress.TimeRemainingColumn(),
- progress.TextColumn("{task.fields[message]}"),
- console=console.Console(),
- transient=False,
- )
- self.progressbar.start()
- self.active_task = None
-
- def __del__(self) -> None:
- # logger.info("Destroying bar...")
- self.progressbar.stop()
-
- @classmethod
- def get_curr_op(cls, op_code: int) -> str:
- """Get OP name from OP code."""
- # Remove BEGIN- and END-flag and get op name
- op_code_masked = op_code & cls.OP_MASK
- return cls.OP_CODE_MAP.get(op_code_masked, "?").title()
-
- def update(
- self,
- op_code: int,
- cur_count: str | float,
- max_count: str | float | None = None,
- message: str | None = None,
- ) -> None:
- # Start new bar on each BEGIN-flag
- if op_code & self.BEGIN:
- self.curr_op = self.get_curr_op(op_code)
- # logger.info("Next: %s", self.curr_op)
- self.active_task = self.progressbar.add_task(
- description=self.curr_op,
- total=max_count,
- message=message,
- )
-
- self.progressbar.update(
- task_id=self.active_task,
- completed=cur_count,
- message=message,
- )
-
- # End progress monitoring on each END-flag
- if op_code & self.END:
- # logger.info("Done: %s", self.curr_op)
- self.progressbar.update(
- task_id=self.active_task,
- message=f"[bright_black]{message}",
- )
-
-
-class ProcessStatus(Enum):
- RUNNING = "[blue]Running"
- DONE = "[green]Done"
- FAILED = "[red]Failed"
-
-
-def docker_desktop_memory() -> int:
- path = str(Path.home()) + "/Library/Group Containers/group.com.docker/settings.json"
-
- try:
- f = open(path)
- out = f.read()
- f.close()
- return json.loads(out)["memoryMiB"]
-
- except Exception: # nosec
- # docker desktop not found - probably running linux
- return -1
-
-
-def asset_path() -> os.PathLike:
- return Path(hagrid_root()) / "hagrid"
-
-
-def manifest_template_path() -> os.PathLike:
- return Path(asset_path()) / "manifest_template.yml"
-
-
-def hagrid_cache_dir() -> os.PathLike:
- return Path("~/.hagrid").expanduser()
-
-
-def repo_src_path() -> Path:
- if EDITABLE_MODE:
- return Path(os.path.abspath(Path(hagrid_root()) / "../../"))
- else:
- return Path(os.path.join(Path(hagrid_cache_dir()) / "PySyft"))
-
-
-def grid_src_path() -> str:
- return str(repo_src_path() / "packages" / "grid")
-
-
-def check_is_git(path: Path) -> bool:
- is_repo = False
- try:
- git.Repo(path)
- is_repo = True
- except Exception: # nosec
- pass
- return is_repo
-
-
-def is_gitpod() -> bool:
- return bool(os.environ.get("GITPOD_WORKSPACE_URL", None))
-
-
-def gitpod_url(port: int | None = None) -> str:
- workspace_url = os.environ.get("GITPOD_WORKSPACE_URL", "")
- if port:
- workspace_url = workspace_url.replace("https://", f"https://{port}-")
- return workspace_url
-
-
-def get_git_repo() -> git.Repo:
- # relative
- from .art import RichEmoji
-
- OK_EMOJI = RichEmoji("white_heavy_check_mark").to_str()
-
- is_git = check_is_git(path=repo_src_path())
- console = rich.get_console()
-
- if not EDITABLE_MODE and not is_git:
- github_repo = "OpenMined/PySyft.git"
- git_url = f"https://github.com/{github_repo}"
-
- print(f"Fetching Syft + Grid Source from {git_url} to {repo_src_path()}")
- try:
- repo_branch = DEFAULT_BRANCH
- repo_path = repo_src_path()
-
- if repo_path.exists():
- shutil.rmtree(str(repo_path))
-
- git.Repo.clone_from(
- git_url,
- str(repo_path),
- single_branch=False,
- b=repo_branch,
- progress=GitRemoteProgress(),
- )
- console.print(f"{OK_EMOJI} Fetched PySyft repo.")
- except Exception as e: # nosec
- print(f"Failed to clone {git_url} to {repo_src_path()} with error: {e}")
- return git.Repo(repo_src_path())
-
-
-def update_repo(repo: git.Repo, branch: str) -> None:
- # relative
- from .art import RichEmoji
-
- OK_EMOJI = RichEmoji("white_heavy_check_mark").to_str()
- console = rich.get_console()
- if not EDITABLE_MODE:
- with console.status("Updating hagrid") as console_status:
- console_status.update(f"[bold blue]Updating HAGrid from branch: {branch}")
- try:
- if repo.is_dirty():
- repo.git.reset("--hard")
- repo.remotes.origin.fetch()
- repo.git.checkout(branch)
- repo.remotes.origin.pull()
- console.print(f"{OK_EMOJI} Updated HAGrid from branch: {branch}")
- except Exception as e:
- print(f"Error checking out branch {branch}.", e)
-
-
-def commit_hash() -> str:
- try:
- repo = get_git_repo()
- sha = repo.head.commit.hexsha
- return sha
- except Exception as e:
- print("failed to get repo sha", e)
- return "unknown"
-
-
-def use_branch(branch: str) -> None:
- if not EDITABLE_MODE:
- print(f"Using HAGrid from branch: {branch}")
- repo = get_git_repo()
- try:
- if repo.is_dirty():
- repo.git.reset("--hard")
- repo.remotes.origin.fetch()
- repo.git.checkout(branch)
- repo.remotes.origin.pull()
- except Exception as e:
- print(f"Error checking out branch {branch}.", e)
-
-
-def should_provision_remote(
- username: str | None, password: str | None, key_path: str | None
-) -> bool:
- is_remote = username is not None or password is not None or key_path is not None
- if username and password or username and key_path:
- return is_remote
- if is_remote:
- raise Exception("--username requires either --password or --key-path")
- return is_remote
-
-
-def name_tag(name: str) -> str:
- return hashlib.sha256(name.encode("utf8")).hexdigest()
-
-
-def find_available_port(
- host: str, port: int | None = None, search: bool = False
-) -> int:
- if port is None:
- port = random.randint(1500, 65000) # nosec
- port_available = False
- while not port_available:
- try:
- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- result_of_check = sock.connect_ex((host, port))
-
- if result_of_check != 0:
- port_available = True
- break
- else:
- if search:
- port += 1
- else:
- break
- sock.close()
-
- except Exception as e:
- print(f"Failed to check port {port}. {e}")
- sock.close()
-
- if search is False and port_available is False:
- error = (
- f"{port} is in use, either free the port or "
- + f"try: {port}+ to auto search for a port"
- )
- raise Exception(error)
- return port
-
-
-def get_version_module() -> tuple[str, str]:
- try:
- version_file_path = f"{grid_src_path()}/VERSION"
- loader = importlib.machinery.SourceFileLoader("VERSION", version_file_path)
- spec = importlib.util.spec_from_loader(loader.name, loader)
- if spec:
- version_module = importlib.util.module_from_spec(spec)
- loader.exec_module(version_module)
- version = version_module.get_version()
- hash = version_module.get_hash()
- return (version, hash)
- except Exception as e:
- print(f"Failed to retrieve versions from: {version_file_path}. {e}")
- return ("unknown", "unknown")
-
-
-# Check base route of an IP address
-def check_host(ip: str, silent: bool = False) -> bool:
- try:
- socket.gethostbyname(ip)
- return True
- except Exception as e:
- if not silent:
- print(f"Failed to resolve host {ip}. {e}")
- return False
-
-
-# Check status of login page
-def check_login_page(ip: str, timeout: int = 30, silent: bool = False) -> bool:
- try:
- url = f"http://{ip}/login"
- response = requests.get(url, timeout=timeout)
- if response.status_code == 200:
- return True
- else:
- return False
- except Exception as e:
- if not silent:
- print(f"Failed to check login page {ip}. {e}")
- return False
-
-
-# Check api metadata
-def check_api_metadata(ip: str, timeout: int = 30, silent: bool = False) -> bool:
- try:
- url = f"http://{ip}/api/v2/metadata"
- response = requests.get(url, timeout=timeout)
- if response.status_code == 200:
- return True
- else:
- return False
- except Exception as e:
- if not silent:
- print(f"Failed to check api metadata {ip}. {e}")
- return False
-
-
-def save_vm_details_as_json(username: str, password: str, process_list: list) -> None:
- """Saves the launched hosts details as json."""
-
- host_ip_details: list = []
-
- # file path to save host details
- dir_path = os.path.expanduser("~/.hagrid")
- os.makedirs(dir_path, exist_ok=True)
- file_path = f"{dir_path}/host_ips.json"
-
- for ip_address, _, jupyter_token in process_list:
- _data = {
- "username": username,
- "password": password,
- "ip_address": ip_address,
- "jupyter_token": jupyter_token,
- }
- host_ip_details.append(_data)
-
- # save host details
- with open(file_path, "w") as fp:
- json.dump({"host_ips": host_ip_details}, fp)
-
- print(f"Saved vm details at: {file_path}")
-
-
-def generate_user_table(username: str, password: str) -> Table | str:
- if not username and not password:
- return ""
-
- table = Table(title="Virtual Machine Credentials")
- table.add_column("Username")
- table.add_column("Password")
-
- table.add_row(f"[green]{username}", f"[green]{password}")
-
- return table
-
-
-def get_process_status(process: subprocess.Popen) -> str:
- poll_status = process.poll()
- if poll_status is None:
- return ProcessStatus.RUNNING.value
- elif poll_status != 0:
- return ProcessStatus.FAILED.value
- else:
- return ProcessStatus.DONE.value
-
-
-def generate_process_status_table(process_list: list) -> tuple[Table, bool]:
- """Generate a table to show the status of the processes being exected.
-
- Args:
- process_list (list): each item in the list
- is a tuple of ip_address, process and jupyter token
-
- Returns:
- Tuple[Table, bool]: table of process status and flag to indicate if all processes are executed.
- """
-
- process_statuses: list[str] = []
- lines_to_display = 5 # Number of lines to display as output
-
- table = Table(title="Virtual Machine Status")
- table.add_column("PID", style="cyan")
- table.add_column("IpAddress", style="magenta")
- table.add_column("Status")
- table.add_column("Jupyter Token", style="white on black")
- table.add_column("Log", overflow="fold", no_wrap=False)
-
- for ip_address, process, jupyter_token in process_list:
- process_status = get_process_status(process)
-
- process_statuses.append(process_status)
-
- process_log = []
- if process_status == ProcessStatus.FAILED.value:
- process_log += process.stderr.readlines(lines_to_display)
- else:
- process_log += process.stdout.readlines(lines_to_display)
-
- process_log_str = "\n".join(log.decode("utf-8") for log in process_log)
- process_log_str = process_log_str if process_log else "-"
-
- table.add_row(
- f"{process.pid}",
- f"{ip_address}",
- f"{process_status}",
- f"{jupyter_token}",
- f"{process_log_str}",
- )
-
- processes_completed = ProcessStatus.RUNNING.value not in process_statuses
-
- return table, processes_completed
-
-
-def check_jupyter_server(
- host_ip: str, wait_time: int = 5, silent: bool = False
-) -> bool:
- if not silent:
- print(f"Checking Jupyter Server at VM {host_ip} is up")
-
- try:
- url = f"http://{host_ip}:8888/"
- response = requests.get(url, timeout=wait_time)
- if response.status_code == 200:
- return True
- else:
- return False
- except Exception as e:
- if not silent:
- print(f"Failed to check jupyter server status {host_ip}. {e}")
- return False
-
-
-GIT_REPO = get_git_repo
-GRID_SRC_VERSION = get_version_module
-GRID_SRC_PATH = grid_src_path
diff --git a/packages/hagrid/hagrid/manifest_template.yml b/packages/hagrid/hagrid/manifest_template.yml
deleted file mode 100644
index eb7c5c9b3d6..00000000000
--- a/packages/hagrid/hagrid/manifest_template.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-manifestVersion: 0.1
-hagrid_version: 0.3.122
-syft_version: 0.8.7-beta.7
-dockerTag: 0.8.7-beta.7
-baseUrl: https://raw.githubusercontent.com/OpenMined/PySyft/
-hash: a4268fab7ad76cd2e854b259660879f15e213824
-target_dir: ~/.hagrid/PySyft/
-files:
- grid:
- path: packages/grid/
- common:
- - default.env
- docker:
- - default.env
- - docker-compose.build.yml
- - docker-compose.dev.yml
- - docker-compose.pull.yml
- - docker-compose.test.yml
- - docker-compose.tls.yml
- - docker-compose.yml
- - traefik/docker/dynamic-tls.yml
- - traefik/docker/dynamic.yml
- - traefik/docker/traefik-tls.template.yml
- - traefik/docker/traefik.yml
- k8s:
- - devspace.yaml
- podman:
- - podman/podman-kube/podman-syft-kube-config.yaml
- - podman/podman-kube/podman-syft-kube.yaml
- - podman/podman-kube/traefik/conf/dynamic.yml
- - podman/podman-kube/traefik/traefik.yml
diff --git a/packages/hagrid/hagrid/mode.py b/packages/hagrid/hagrid/mode.py
deleted file mode 100644
index e21da8ccbba..00000000000
--- a/packages/hagrid/hagrid/mode.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# stdlib
-import os
-from pathlib import Path
-import site
-
-
-def str_to_bool(bool_str: str | None) -> bool:
- result = False
- bool_str = str(bool_str).lower()
- if bool_str == "true" or bool_str == "1":
- result = True
- return result
-
-
-def hagrid_root() -> str:
- return os.path.abspath(str(Path(__file__).parent.parent))
-
-
-def is_editable_mode() -> bool:
- disable_editable_mode = str_to_bool(
- os.environ.get("DISABLE_EDITABLE_MODE", "False")
- )
- if disable_editable_mode:
- print("šØ Editable Mode DISABLED")
- return False
- current_package_root = hagrid_root()
-
- installed_as_editable = False
- sitepackages_dirs = site.getsitepackages()
- # check all site-packages returned if they have a hagrid.egg-link
- for sitepackages_dir in sitepackages_dirs:
- egg_link_file = Path(sitepackages_dir) / "hagrid.egg-link"
- try:
- linked_folder = egg_link_file.read_text()
- # if the current code is in the same path as the egg-link its -e mode
- installed_as_editable = current_package_root in linked_folder
- break
- except Exception: # nosec
- pass
-
- if os.path.exists(Path(current_package_root) / "hagrid.egg-info"):
- installed_as_editable = True
-
- return installed_as_editable
-
-
-EDITABLE_MODE = is_editable_mode()
diff --git a/packages/hagrid/hagrid/names.py b/packages/hagrid/hagrid/names.py
deleted file mode 100644
index 26e6b92025d..00000000000
--- a/packages/hagrid/hagrid/names.py
+++ /dev/null
@@ -1,185 +0,0 @@
-# stdlib
-from secrets import randbelow
-
-left_name = [
- "admiring",
- "adoring",
- "affectionate",
- "agitated",
- "amazing",
- "angry",
- "awesome",
- "beautiful",
- "blissful",
- "bold",
- "boring",
- "brave",
- "busy",
- "charming",
- "clever",
- "cool",
- "compassionate",
- "competent",
- "condescending",
- "confident",
- "cranky",
- "crazy",
- "dazzling",
- "determined",
- "distracted",
- "dreamy",
- "eager",
- "ecstatic",
- "elastic",
- "elated",
- "elegant",
- "eloquent",
- "epic",
- "exciting",
- "fervent",
- "festive",
- "flamboyant",
- "focused",
- "friendly",
- "frosty",
- "funny",
- "gallant",
- "gifted",
- "goofy",
- "gracious",
- "great",
- "happy",
- "hardcore",
- "heuristic",
- "hopeful",
- "hungry",
- "infallible",
- "inspiring",
- "interesting",
- "intelligent",
- "jolly",
- "jovial",
- "keen",
- "kellis",
- "kind",
- "laughing",
- "loving",
- "lucid",
- "magical",
- "mystifying",
- "modest",
- "musing",
- "naughty",
- "nervous",
- "nice",
- "nifty",
- "nostalgic",
- "objective",
- "optimistic",
- "peaceful",
- "pedantic",
- "pensive",
- "practical",
- "priceless",
- "quirky",
- "quizzical",
- "recursing",
- "relaxed",
- "reverent",
- "romantic",
- "sad",
- "serene",
- "sharp",
- "silly",
- "sleepy",
- "stoic",
- "strange",
- "stupefied",
- "suspicious",
- "sweet",
- "tender",
- "thirsty",
- "trusting",
- "unruffled",
- "upbeat",
- "vibrant",
- "vigilant",
- "vigorous",
- "wizardly",
- "wonderful",
- "xenodochial",
- "youthful",
- "zealous",
- "zen",
-]
-
-right_name = [
- "altman",
- "bach",
- "bengios",
- "bostrom",
- "botvinick",
- "brockman",
- "chintala",
- "chollet",
- "chomsky",
- "dean",
- "dolgov",
- "eckersley",
- "fridman",
- "gardner",
- "goertzel",
- "goodfellow",
- "hassabis",
- "he",
- "hinton",
- "hochreiter",
- "hotz",
- "howard",
- "hutter",
- "isbell",
- "kaliouby",
- "karp",
- "karpathy",
- "kearns",
- "kellis",
- "knuth",
- "koller",
- "krizhevsky",
- "larochelle",
- "lattner",
- "lecun",
- "li",
- "lim",
- "littman",
- "malik",
- "mironov",
- "ng",
- "norvig",
- "olah",
- "pearl",
- "pesenti",
- "russell",
- "salakhutdinov",
- "schmidhuber",
- "silver",
- "smola",
- "song",
- "sophia",
- "sutskever",
- "thomas",
- "thrun",
- "trask",
- "vapnik",
- "vaswani",
- "vinyals",
- "winston",
- "wolf",
- "wolfram",
-]
-
-
-def random_name() -> str:
- left_i = randbelow(len(left_name) - 1)
- right_i = randbelow(len(right_name) - 1)
- return f"{left_name[left_i].capitalize()} {right_name[right_i].capitalize()}"
diff --git a/packages/hagrid/hagrid/nb_output.py b/packages/hagrid/hagrid/nb_output.py
deleted file mode 100644
index c0a4bb0d2fb..00000000000
--- a/packages/hagrid/hagrid/nb_output.py
+++ /dev/null
@@ -1,15 +0,0 @@
-# future
-from __future__ import annotations
-
-
-# alert-info, alert-warning, alert-success, alert-danger
-class NBOutput:
- def __init__(self, raw_output: str) -> None:
- self.raw_output = raw_output
-
- def _repr_html_(self) -> str:
- return self.raw_output
-
- def to_html(self) -> NBOutput:
- self.raw_output = self.raw_output.replace("\n", "Please go through the Install Wizard notebook to " - + "install Syft and optionally start a Grid server." - ) - html += ( - '
š Click to start ' - + "Install š§š½āāļø Wizard
Try running:' - ) - - return html - - -def get_urls_from_dir( - url: str, - repo: str, - branch: str, - commit: str | None = None, -) -> list[str]: - notebooks = [] - slug = commit if commit else branch - - gh_api_call = ( - "https://api.github.com/repos/" + repo + "/git/trees/" + slug + "?recursive=1" - ) - r = requests.get(gh_api_call) # nosec - if r.status_code != 200: - print( - f"Failed to fetch notebook from: {gh_api_call}.\n" - + "Please try again with the correct parameters!" - ) - sys.exit(1) - - res = r.json() - - for file in res["tree"]: - if file["path"].startswith("notebooks/quickstart/" + url) or file[ - "path" - ].startswith("notebooks/" + url): - if file["path"].endswith(".ipynb"): - temp_url = ( - "https://raw.githubusercontent.com/" - + repo - + "/" - + slug - + "/" - + file["path"] - ) - notebooks.append(temp_url) - - if len(notebooks) == 0: - for file in res["tree"]: - if file["path"].startswith("notebooks/" + url): - if file["path"].endswith(".ipynb"): - temp_url = ( - "https://raw.githubusercontent.com/" - + repo - + "/" - + slug - + "/" - + file["path"] - ) - notebooks.append(temp_url) - return notebooks diff --git a/packages/hagrid/hagrid/rand_sec.py b/packages/hagrid/hagrid/rand_sec.py deleted file mode 100644 index 3323554a72f..00000000000 --- a/packages/hagrid/hagrid/rand_sec.py +++ /dev/null @@ -1,84 +0,0 @@ -# stdlib -from os import urandom -import string -import sys - - -def generate_sec_random_password( - length: int, - special_chars: bool = True, - digits: bool = True, - lower_case: bool = True, - upper_case: bool = True, -) -> str: - """Generates a random password of the given length. - - Args: - length (int): length of the password - special_chars (bool, optional): Include at least one specials char in the password. Defaults to True. - digits (bool, optional): Include at least one digit in the password. Defaults to True. - lower_case (bool, optional): Include at least one lower case character in the password. Defaults to True. - upper_case (bool, optional): Includde at least one upper case character in the password. Defaults to True. - - Raises: - ValueError: If password length if too short. - - Returns: - str: randomly generated password - """ - if not isinstance(length, int) or length < 10: - raise ValueError( - "Password should have a positive safe length of at least 10 characters!" - ) - - choices: str = "" - required_tokens: list[str] = [] - if special_chars: - special_characters = "!@#$%^&*()_+" - choices += special_characters - required_tokens.append( - special_characters[ - int.from_bytes(urandom(1), sys.byteorder) % len(special_characters) - ] - ) - if lower_case: - choices += string.ascii_lowercase - required_tokens.append( - string.ascii_lowercase[ - int.from_bytes(urandom(1), sys.byteorder) % len(string.ascii_lowercase) - ] - ) - if upper_case: - choices += string.ascii_uppercase - required_tokens.append( - string.ascii_uppercase[ - int.from_bytes(urandom(1), sys.byteorder) % len(string.ascii_uppercase) - ] - ) - if digits: - choices += string.digits - required_tokens.append( - string.digits[ - int.from_bytes(urandom(1), sys.byteorder) % len(string.digits) - ] - ) - - # Python 3 (urandom returns bytes) - password = [choices[c % len(choices)] for c in urandom(length)] - - # Pick some random indexes - random_indexes: set[int] = set() - while len(random_indexes) < len(required_tokens): - random_indexes.add(int.from_bytes(urandom(1), sys.byteorder) % len(password)) - - # Replace the random indexes with the required tokens - for i, idx in enumerate(random_indexes): - password[idx] = required_tokens[i] - - return "".join(password) - - -if __name__ == "__main__": - pwd_length = 48 - # generate_sec_random_password(pwd_length) - print(generate_sec_random_password(pwd_length, special_chars=False)) diff --git a/packages/hagrid/hagrid/stable_version.py b/packages/hagrid/hagrid/stable_version.py deleted file mode 100644 index d596aae77cd..00000000000 --- a/packages/hagrid/hagrid/stable_version.py +++ /dev/null @@ -1 +0,0 @@ -LATEST_STABLE_SYFT = "0.8.6" diff --git a/packages/hagrid/hagrid/style.py b/packages/hagrid/hagrid/style.py deleted file mode 100644 index f42d9f79061..00000000000 --- a/packages/hagrid/hagrid/style.py +++ /dev/null @@ -1,44 +0,0 @@ -# stdlib -import io - -# third party -import click -import rich - -# relative -from .deps import DEPENDENCIES -from .mode import EDITABLE_MODE - - -class RichGroup(click.Group): - def format_usage( - self, ctx: click.core.Context, formatter: click.formatting.HelpFormatter - ) -> None: - sio = io.StringIO() - console = rich.get_console() - mode = "" - if EDITABLE_MODE: - mode = "[bold red]EDITABLE DEV MODE[/bold red] :police_car_light:" - console.print( - "[bold red]HA[/bold red][bold magenta]Grid[/bold magenta]!", ":mage:", mode - ) - table = rich.table.Table() - - table.add_column("Dependency", style="magenta") - table.add_column("Found", justify="right") - - for dep in sorted(DEPENDENCIES.keys()): - path = DEPENDENCIES[dep] - installed_str = ":white_check_mark:" if path is not None else ":cross_mark:" - dep_emoji = ":gear:" - if dep == "docker": - dep_emoji = ":whale:" - if dep == "git": - dep_emoji = ":file_folder:" - if dep == "ansible-playbook": - dep_emoji = ":blue_book:" - table.add_row(f"{dep_emoji} {dep}", installed_str) - # console.print(dep_emoji, dep, installed_str) - console.print(table) - console.print("Usage: hagrid [OPTIONS] COMMAND [ARGS]...") - formatter.write(sio.getvalue()) diff --git a/packages/hagrid/hagrid/util.py b/packages/hagrid/hagrid/util.py deleted file mode 100644 index 73d1cf1e34e..00000000000 --- a/packages/hagrid/hagrid/util.py +++ /dev/null @@ -1,102 +0,0 @@ -# stdlib -from collections.abc import Callable -from enum import Enum -import os -import subprocess # nosec -import sys -from typing import Any -from urllib.parse import urlparse - -# relative -from .dummynum import DummyNum - - -class NodeSideType(str, Enum): - LOW_SIDE = "low" - HIGH_SIDE = "high" - - def __str__(self) -> str: - # Use values when transforming NodeType to str - return self.value - - -class ImportFromSyft: - @staticmethod - def import_syft_error() -> Callable: - try: - # syft absolute - from syft.service.response import SyftError - except Exception: - SyftError = DummyNum - - return SyftError - - @staticmethod - def import_stage_protocol_changes() -> Callable: - try: - # syft absolute - from syft.protocol.data_protocol import stage_protocol_changes - except Exception: - - def stage_protocol_changes(*args: Any, **kwargs: Any) -> None: - pass - - return stage_protocol_changes - - @staticmethod - def import_node_type() -> Callable: - try: - # syft absolute - from syft.abstract_node import NodeType - except Exception: - NodeType = DummyNum - - return NodeType - - -def from_url(url: str) -> tuple[str, str, int, str, Any | str]: - try: - # urlparse doesnt handle no protocol properly - if "://" not in url: - url = "http://" + url - parts = urlparse(url) - host_or_ip_parts = parts.netloc.split(":") - # netloc is host:port - port = 80 - if len(host_or_ip_parts) > 1: - port = int(host_or_ip_parts[1]) - host_or_ip = host_or_ip_parts[0] - return ( - host_or_ip, - parts.path, - port, - parts.scheme, - getattr(parts, "query", ""), - ) - except Exception as e: - print(f"Failed to convert url: {url} to GridURL. {e}") - raise e - - -def fix_windows_virtualenv_api(cls: type) -> None: - # fix bug in windows - def _python_rpath(self: Any) -> str: - """The relative path (from environment root) to python.""" - # Windows virtualenv installation installs pip to the [Ss]cripts - # folder. Here's a simple check to support: - if sys.platform == "win32": - # fix here https://github.com/sjkingo/virtualenv-api/issues/47 - return os.path.join(self.path, "Scripts", "python.exe") - return os.path.join("bin", "python") - - cls._python_rpath = property(_python_rpath) - - -def shell(command: str) -> str: - try: - output = subprocess.check_output( # nosec - command, shell=True, stderr=subprocess.STDOUT - ) - except Exception: - output = b"" - return output.decode("utf-8") diff --git a/packages/hagrid/hagrid/version.py b/packages/hagrid/hagrid/version.py deleted file mode 100644 index 6d599e5c8e7..00000000000 --- a/packages/hagrid/hagrid/version.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python3 -# HAGrid Version -__version__ = "0.3.122" - -if __name__ == "__main__": - print(__version__) diff --git a/packages/hagrid/hagrid/win_bootstrap.py b/packages/hagrid/hagrid/win_bootstrap.py deleted file mode 100644 index 9cd79c24c36..00000000000 --- a/packages/hagrid/hagrid/win_bootstrap.py +++ /dev/null @@ -1,267 +0,0 @@ -# stdlib -from collections.abc import Callable -import subprocess # nosec - -# one liner to use bootstrap script: -# CMD: curl https://raw.githubusercontent.com/OpenMined/PySyft/dev/packages/hagrid/hagrid/win_bootstrap.py > win_bootstrap.py && python win_bootstrap.py # noqa -# Powershell is complaining about a utf-8 issue we need to fix, could be related to a -# bug with long lines in utf-8 -# PS: $r = Invoke-WebRequest "https://raw.githubusercontent.com/OpenMined/PySyft/dev/packages/hagrid/hagrid/win_bootstrap.py" -UseBasicParsing; echo $r.Content > win_bootstrap.py; python win_bootstrap.py # noqa - - -class Requirement: - def __init__( - self, full_name: str, choco_name: str, detect: Callable, extras: str = "" - ) -> None: - self.full_name = full_name - self.choco_name = choco_name - self.detect = detect - self.extras = extras - - def __repr__(self) -> str: - return self.full_name - - -install_choco_pwsh = """ -[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; -Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')); -""" - -install_wsl2_pwsh = """ -wsl --update; wsl --shutdown; wsl --set-default-version 2; wsl --install -d Ubuntu; wsl --setdefault Ubuntu; -""" - -# add this to block powershell from existing for debugging -# Read-Host -Prompt string - - -def make_admin_cmd(admin_cmd: str) -> str: - return ( - f"Start-Process PowerShell -Wait -Verb RunAs -ArgumentList " - '"' - "Set-ExecutionPolicy Bypass -Scope Process -Force; " - f"{admin_cmd}; " - '"' - ) - - -def where_is(binary: str, req: Requirement) -> bool: - print(f"{req.full_name} - {binary}", end="", flush=True) - found = path_where_is(binary) - if not found: - found = full_where_is(binary) - if found: - print(" ā") - else: - print(" Ć") - return found - - -def path_where_is(binary: str) -> bool: - try: - cmds = ["where.exe", binary] - output = subprocess.run(cmds, capture_output=True, cwd="C:\\") # nosec - out = str(output.stdout.decode("utf-8")).split("\r\n") - if binary in out[0]: - return True - except Exception as e: - print("error", e) - pass - return False - - -def full_where_is(binary: str) -> bool: - try: - powershell_cmd = f"where.exe /R C:\ *.exe | findstr \\{binary}$" # noqa: W605 - cmds = ["powershell.exe", "-Command", powershell_cmd] - output = subprocess.run(cmds, capture_output=True, cwd="C:\\") # nosec - out = str(output.stdout.decode("utf-8")).split("\r\n") - if binary in out[0]: - return True - except Exception as e: - print("error", e) - pass - return False - - -def exe(binary: str) -> Callable: - def call(req: Requirement) -> bool: - return where_is(binary=binary, req=req) - - return call - - -def detect_wsl2(req: Requirement) -> bool: - print(f"{req.full_name} - wsl.exe ", end="") - try: - powershell_cmd = "wsl.exe --status" - cmds = ["powershell.exe", "-Command", powershell_cmd] - output = subprocess.run(cmds, capture_output=True) # nosec - out = output.stdout.decode("utf-16") - if "Default Distribution: Ubuntu" in out: - pass - if "Default Version: 2" in out: - print(" ā") - return True - except Exception as e: - print("error", e) - pass - print(" Ć") - return False - - -requirements = [] -requirements.append( - Requirement( - full_name="Windows Subsystem for Linux 2", - choco_name="wsl2", - detect=detect_wsl2, - ) -) -requirements.append( - Requirement( - full_name="Chocolatey Package Manager", - choco_name="choco", - detect=exe("choco.exe"), - ) -) -requirements.append( - Requirement( - full_name="Anaconda Individual Edition", - choco_name="anaconda3", - detect=exe("conda.exe"), - ) -) -requirements.append( - Requirement( - full_name="Git Version Control", - choco_name="git", - detect=exe("git.exe"), - ) -) -requirements.append( - Requirement( - full_name="Docker Desktop", - choco_name="docker-desktop", - detect=exe("docker.exe"), - ) -) - - -def install_elevated_powershell(full_name: str, powershell_cmd: str) -> None: - try: - input( - f"\nInstalling {full_name} requires Administrator.\n" - "When the UAC dialogue appears click Yes on the left.\n\n" - "Press enter to start..." - ) - powershell_cmds = ["-command", powershell_cmd] - output = subprocess.run( # nosec - ["powershell.exe"] + powershell_cmds, capture_output=True - ) - _ = output.stdout.decode("utf-8") - except Exception as e: - print("failed", e) - - -def install_choco() -> None: - return install_elevated_powershell( - full_name="Chocolatey", powershell_cmd=make_admin_cmd(install_choco_pwsh) - ) - - -def install_wsl2() -> None: - return install_elevated_powershell( - full_name="WSL2", powershell_cmd=make_admin_cmd(install_wsl2_pwsh) - ) - - -def install_deps(requirements: list[Requirement]) -> None: - package_names = [] - for req in requirements: - package_names.append(req.choco_name) - - try: - input( - "\nInstalling packages requires Administrator.\n" - "When the UAC dialogue appears click Yes on the left.\n\n" - "Press enter to start..." - ) - choco_args = f"choco.exe install {' '.join(package_names)} -y" - powershell_cmds = ["-command", make_admin_cmd(choco_args)] - output = subprocess.run( # nosec - ["powershell.exe"] + powershell_cmds, capture_output=True - ) - _ = str(output.stdout.decode("utf-8")) - except Exception as e: - print("failed", e) - - -def ask_install(requirement: Requirement) -> bool: - val = input(f"Do you want to install {requirement.full_name} (Y/n): ") - if "y" in val.lower(): - return True - return False - - -def check_all(requirements: list[Requirement]) -> list[Requirement]: - missing = [] - for req in requirements: - if not req.detect(req): - missing.append(req) - return missing - - -def main() -> None: - print("\nHAGrid Windows Dependency Installer") - print("===================================\n") - print("Searching your computer for:") - missing_deps = check_all(requirements=requirements) - - if len(missing_deps) > 0: - print("\nWe were unable to find the following dependencies:") - print("-----------------------------------") - for dep in missing_deps: - print(f"{dep.full_name}") - - print("") - desired = [] - choco_required = False - wsl2_required = False - for dep in missing_deps: - if ask_install(dep): - if dep.choco_name == "choco": - choco_required = True - elif dep.choco_name == "wsl2": - wsl2_required = True - else: - desired.append(dep) - elif dep.choco_name == "choco": - print("You must install Chocolatey to install other dependencies") - return - - if wsl2_required: - install_wsl2() - - if choco_required: - install_choco() - - if len(desired) > 0: - install_deps(desired) - - print("") - still_missing = check_all(requirements=missing_deps) - if len(still_missing) > 0: - print("We were still unable to find the following dependencies:") - print("-----------------------------------") - for dep in still_missing: - print(f"{dep.full_name}") - print("Please try again.") - else: - print("\nCongratulations. All done.") - print("===================================\n") - print("Now you can run HAGrid on Windows!") - - -if __name__ == "__main__": - main() diff --git a/packages/hagrid/hagrid/wizard_ui.py b/packages/hagrid/hagrid/wizard_ui.py deleted file mode 100644 index 7f4c5c1c0d4..00000000000 --- a/packages/hagrid/hagrid/wizard_ui.py +++ /dev/null @@ -1,62 +0,0 @@ -# stdlib - -# relative -from .cache import arg_cache -from .deps import Dependency -from .deps import check_grid_docker -from .deps import check_hagrid -from .deps import check_syft -from .deps import check_syft_deps -from .nb_output import NBOutput - -steps = {} -steps["check_hagrid"] = False -steps["check_syft"] = False -steps["check_grid"] = False - - -def complete_install_wizard( - output: dict[str, Dependency] | NBOutput, -) -> dict[str, Dependency] | NBOutput: - flipped = arg_cache["install_wizard_complete"] - if not flipped: - for _, v in steps.items(): - if v is False: - return output - arg_cache["install_wizard_complete"] = True - if isinstance(output, NBOutput): - if flipped != arg_cache["install_wizard_complete"]: - output.raw_output += "\n\nā You have completed the Install Wizard" - return output - - -class WizardUI: - @property - def check_hagrid(self) -> dict[str, Dependency] | NBOutput: - steps["check_hagrid"] = True - return complete_install_wizard(check_hagrid()) - - @property - def check_syft_deps(self) -> dict[str, Dependency] | NBOutput: - steps["check_syft"] = True - return complete_install_wizard(check_syft_deps()) - - @property - def check_syft(self) -> dict[str, Dependency] | NBOutput: - steps["check_syft"] = True - return complete_install_wizard(check_syft()) - - @property - def check_syft_pre(self) -> dict[str, Dependency] | NBOutput: - steps["check_syft"] = True - return complete_install_wizard(check_syft(pre=True)) - - @property - def check_grid_docker(self) -> dict[str, Dependency] | NBOutput: - print("Deprecated. Please use .check_docker") - return self.check_docker - - @property - def check_docker(self) -> dict[str, Dependency] | NBOutput: - steps["check_grid"] = True - return complete_install_wizard(check_grid_docker()) diff --git a/packages/hagrid/scripts/install.sh b/packages/hagrid/scripts/install.sh deleted file mode 100755 index aa13f184093..00000000000 --- a/packages/hagrid/scripts/install.sh +++ /dev/null @@ -1,232 +0,0 @@ -#!/bin/sh - -# run with: -# curl https://raw.githubusercontent.com/OpenMined/PySyft/dev/packages/hagrid/scripts/install.sh | sh - -set -e - -cat /dev/null <" - + f'quickstart.download("{first}")