@@ -123,7 +167,7 @@ def to_html(self) -> str:
""" # noqa: E501
diff --git a/packages/syft/src/syft/util/util.py b/packages/syft/src/syft/util/util.py
index fd2c4dc4074..b0affa2b1a0 100644
--- a/packages/syft/src/syft/util/util.py
+++ b/packages/syft/src/syft/util/util.py
@@ -19,6 +19,7 @@
import os
from pathlib import Path
import platform
+import random
import re
from secrets import randbelow
import socket
@@ -309,7 +310,11 @@ def print_dynamic_log(
return (finish, success)
-def find_available_port(host: str, port: int, search: bool = False) -> int:
+def find_available_port(
+ host: str, port: int | None = None, search: bool = False
+) -> int:
+ if port is None:
+ port = random.randint(1500, 65000) # nosec
port_available = False
while not port_available:
try:
@@ -324,6 +329,7 @@ def find_available_port(host: str, port: int, search: bool = False) -> int:
port += 1
else:
break
+ sock.close()
except Exception as e:
print(f"Failed to check port {port}. {e}")
@@ -338,6 +344,18 @@ def find_available_port(host: str, port: int, search: bool = False) -> int:
return port
+def get_random_available_port() -> int:
+ """Retrieve a random available port number from the host OS.
+
+ Returns
+ -------
+ int: Available port number.
+ """
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as soc:
+ soc.bind(("localhost", 0))
+ return soc.getsockname()[1]
+
+
def get_loaded_syft() -> ModuleType:
return sys.modules[__name__.split(".")[0]]
diff --git a/packages/syft/tests/syft/project/project_test.py b/packages/syft/tests/syft/project/project_test.py
index 9b2c8ce92f3..c186f2f35fa 100644
--- a/packages/syft/tests/syft/project/project_test.py
+++ b/packages/syft/tests/syft/project/project_test.py
@@ -23,7 +23,7 @@ def test_project_creation(worker):
name="My Cool Project", description="My Cool Description", members=[ds_client]
)
- project = new_project.start()
+ project = new_project.send()
assert isinstance(project, Project)
assert new_project.id == project.id
@@ -47,7 +47,7 @@ def test_error_data_owner_project_creation(worker):
name="My Cool Project", description="My Cool Description", members=[root_client]
)
- project = new_project.start()
+ project = new_project.send()
assert isinstance(project, sy.SyftError)
assert project.message == "Only Data Scientists can create projects"
@@ -96,7 +96,7 @@ def test_project_serde(worker):
name="My Cool Project", description="My Cool Description", members=[root_client]
)
- project = new_project.start()
+ project = new_project.send()
ser_data = sy.serialize(project, to_bytes=True)
assert isinstance(ser_data, bytes)
diff --git a/packages/syft/tests/syft/service/jobs/job_stash_test.py b/packages/syft/tests/syft/service/jobs/job_stash_test.py
index 9d3f5a964aa..271634e1f56 100644
--- a/packages/syft/tests/syft/service/jobs/job_stash_test.py
+++ b/packages/syft/tests/syft/service/jobs/job_stash_test.py
@@ -1,6 +1,7 @@
# stdlib
from datetime import datetime
from datetime import timedelta
+from datetime import timezone
# third party
import pytest
@@ -33,7 +34,7 @@ def test_eta_string(current_iter, n_iters, status, creation_time_delta, expected
node_uid=UID(),
n_iters=n_iters,
current_iter=current_iter,
- creation_time=(datetime.now() - creation_time_delta).isoformat(),
+ creation_time=(datetime.now(tz=timezone.utc) - creation_time_delta).isoformat(),
status=status,
)
diff --git a/packages/syft/tests/syft/zmq_queue_test.py b/packages/syft/tests/syft/zmq_queue_test.py
index 9b22ac7d260..8c5b8dedebe 100644
--- a/packages/syft/tests/syft/zmq_queue_test.py
+++ b/packages/syft/tests/syft/zmq_queue_test.py
@@ -21,9 +21,7 @@
from syft.service.response import SyftError
from syft.service.response import SyftSuccess
from syft.util.util import get_queue_address
-
-# relative
-from ..utils.random_port import get_random_port
+from syft.util.util import get_random_available_port
@pytest.fixture
@@ -118,7 +116,7 @@ def handle_message(message: bytes, *args, **kwargs):
@pytest.fixture
def producer():
- pub_port = get_random_port()
+ pub_port = get_random_available_port()
QueueName = token_hex(8)
# Create a producer
diff --git a/packages/syft/tests/utils/random_port.py b/packages/syft/tests/utils/random_port.py
deleted file mode 100644
index c3370694afb..00000000000
--- a/packages/syft/tests/utils/random_port.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# stdlib
-import socket
-
-
-def get_random_port():
- soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- soc.bind(("", 0))
- return soc.getsockname()[1]
diff --git a/scripts/aa_demo/update_domain.sh b/scripts/aa_demo/update_domain.sh
deleted file mode 100644
index 6efa4106b23..00000000000
--- a/scripts/aa_demo/update_domain.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# $1 domain ip
-# $2 dataset url
-# install syft in dev mode
-pip install -U -e packages/syft
-
-# get domain name
-NODE_NAME="$(sudo docker ps --format '{{.Names}}' | grep "celery" |rev | cut -c 16- | rev)"
-echo "Domain Name: ${NODE_NAME}"
-echo "Nuking Domain .. >:)";
-
-# destroy current domain
-hagrid land all
-echo "Launching Domain .. hahaha >:)";
-
-# re-launch domain
-hagrid launch ${NODE_NAME} to docker:80 --dev --build-src="model_training_tests"
-
-# wait for domain to be up
-hagrid check --timeout=120
-
-echo "Domain lauch succeeded."
-echo "Starting to upload dataset"
-
-# upload dataset
-python scripts/aa_demo/upload_dataset.py $1 $2
-
-echo "Upload dataset script complete."
diff --git a/scripts/hagrid_hash b/scripts/hagrid_hash
deleted file mode 100644
index 715b59990cc..00000000000
--- a/scripts/hagrid_hash
+++ /dev/null
@@ -1 +0,0 @@
-56f89d45a711a6bf79a460fc8cd4ae20
diff --git a/scripts/staging.json b/scripts/staging.json
deleted file mode 100644
index f3b648cff6f..00000000000
--- a/scripts/staging.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "domain-staging-dev": {
- "name": "domain-staging-dev",
- "node_type": "domain",
- "ip": "40.78.86.246",
- "metadata_endpoint": "/api/v1/new/metadata",
- "branch": "dev",
- "commit_hash": "6bd3bea303107f8eef9ad0e43c85006161835fc9"
- }
-}
diff --git a/scripts/staging.py b/scripts/staging.py
deleted file mode 100644
index e158c72b30d..00000000000
--- a/scripts/staging.py
+++ /dev/null
@@ -1,201 +0,0 @@
-# stdlib
-import json
-import os
-import subprocess
-from typing import Any
-
-# third party
-import git
-import requests
-
-DEV_MODE = False
-KEY = None
-JSON_DATA = os.path.dirname(__file__) + "/staging.json"
-
-
-def run_hagrid(node: dict) -> int:
- name = node["name"]
- node_type = node["node_type"]
- ip = node["ip"]
- branch = node["branch"]
- cmd = (
- f"hagrid launch {name} {node_type} to {ip} --username=azureuser --auth-type=key "
- f"--key-path={KEY} --repo=OpenMined/PySyft --branch={branch} --verbose"
- )
- watch_shell(cmd)
-
-
-def watch_shell(command: str) -> None:
- process = subprocess.Popen(
- command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
- ) # nosec
- while True:
- output = process.stdout.readline().decode()
- if output == "" and process.poll() is not None:
- break
- if output:
- print(output.strip())
- rc = process.poll()
- return rc
-
-
-def shell(command: str) -> str:
- try:
- output = subprocess.check_output( # nosec
- command, shell=True, stderr=subprocess.STDOUT
- )
- except Exception:
- output = b""
- return output.decode("utf-8").strip()
-
-
-def metadata_url(node: dict) -> str:
- ip = node["ip"]
- endpoint = node["metadata_endpoint"]
- return f"http://{ip}{endpoint}"
-
-
-def check_metadata(node: dict) -> dict | None:
- try:
- res = requests.get(metadata_url(node))
- if res.status_code != 200:
- print(f"Got status_code: {res.status_code}")
- metadata = res.json()
- name = node["name"]
- print(f"{name} syft_version: ", metadata["syft_version"])
- return metadata
- except Exception as e:
- print(f"Failed to get metadata. {e}")
- return None
-
-
-def process_node(node: dict[str, Any]) -> tuple[bool, str]:
- repo_hash = get_repo_checkout(node)
- metadata = check_metadata(node)
- hash_string = check_remote_hash(node)
- redeploy = False
- if metadata is None or hash_string is None:
- print(f"redeploy because metadata: {metadata} and remote hash: {hash_string}")
- redeploy = True
-
- if hash_string is not None and repo_hash != hash_string:
- print("repo_hash", len(repo_hash), type(repo_hash))
- print("hash_string", len(hash_string), type(hash_string))
- print(
- f"redeploy because repo_hash: {repo_hash} != remote hash_string: {hash_string}"
- )
- redeploy = True
-
- if redeploy:
- print("π§ Redeploying with HAGrid")
- run_hagrid(node)
-
- hash_string = check_remote_hash(node)
- if hash_string is None:
- print(f"Cant get hash: {hash_string}")
-
- if hash_string is not None and hash_string != repo_hash:
- print(
- f"Hash doesnt match repo_hash: {repo_hash} != remote hash_string {hash_string}"
- )
-
- metadata = check_metadata(node)
- if metadata is None:
- print(f"Cant get metadata: {metadata}")
-
- if metadata and hash_string == repo_hash:
- return True, repo_hash
- return False, repo_hash
-
-
-def get_repo_checkout(node: dict) -> str:
- try:
- branch = node["branch"]
- repo_path = f"/tmp/{branch}/PySyft"
- if not os.path.exists(repo_path):
- os.makedirs(repo_path, exist_ok=True)
- repo = git.Repo.clone_from(
- "https://github.com/OpenMined/pysyft",
- repo_path,
- single_branch=True,
- b=branch,
- )
- else:
- repo = git.Repo(repo_path)
- if repo.is_dirty():
- repo.git.reset("--hard")
- repo.git.checkout(branch)
- repo.remotes.origin.pull()
- sha = repo.head.commit.hexsha
- return sha
- except Exception as e:
- print(f"Failed to get branch HEAD commit hash. {e}")
- raise e
-
-
-def run_remote_shell(node: dict, cmd: str) -> str | None:
- try:
- ip = node["ip"]
- ssh_cmd = (
- f"ssh -o LogLevel=ERROR -o StrictHostKeyChecking=no -i {KEY} azureuser@{ip}"
- )
- shell_cmd = f'{ssh_cmd} "{cmd}"'
- print("Running:", shell_cmd)
- return shell(shell_cmd)
- except Exception:
- print("Failed to run ssh command: {}")
- return None
-
-
-def check_remote_hash(node: dict) -> str | None:
- cmd = "sudo runuser -l om -c 'cd /home/om/PySyft && git rev-parse HEAD'"
- return run_remote_shell(node, cmd)
-
-
-def check_staging() -> None:
- nodes = load_staging_data(JSON_DATA)
- for name, node in nodes.items():
- print(f"Processing {name}")
- good = False
- try:
- good, updated_hash = process_node(node=node)
- node["commit_hash"] = updated_hash
- nodes[name] = node
- save_staging_data(JSON_DATA, nodes)
- except Exception as e:
- print(f"Failed to process node: {name}. {e}")
- emoji = "β
" if good else "β"
- print(f"{emoji} Node {name}")
-
-
-def load_staging_data(path: str) -> dict[str, dict]:
- with open(path) as f:
- return json.loads(f.read())
-
-
-def save_staging_data(path: str, data: dict[str, dict]) -> None:
- print("Saving changes to file", path)
- with open(path, "w") as f:
- f.write(f"{json.dumps(data)}")
-
-
-if __name__ == "__main__":
- # stdlib
- import argparse
-
- parser = argparse.ArgumentParser()
- parser.add_argument("--dev", action="store_true", help="Dev Mode")
- parser.add_argument("--private-key", help="Dev Mode")
-
- args = parser.parse_args()
- if args.dev:
- DEV_MODE = True
- if args.private_key:
- path = os.path.expanduser(args.private_key)
- if os.path.exists(path):
- KEY = path
- if KEY is None:
- raise Exception("--private-key required")
- print("DEV MODE", DEV_MODE)
-
- check_staging()
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index f6ccf94f32c..9152038b1f7 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -51,7 +51,6 @@ def full_low_worker(n_consumers: int = 3, create_producer: bool = True) -> Worke
n_consumers=n_consumers,
create_producer=create_producer,
queue_port=None,
- in_memory_workers=True,
local_db=False,
thread_workers=False,
)
@@ -72,7 +71,6 @@ def full_high_worker(n_consumers: int = 3, create_producer: bool = True) -> Work
n_consumers=n_consumers,
create_producer=create_producer,
queue_port=None,
- in_memory_workers=True,
local_db=False,
thread_workers=False,
)
diff --git a/tests/integration/container_workload/pool_image_test.py b/tests/integration/container_workload/pool_image_test.py
index d8cb59f8f1f..96613240660 100644
--- a/tests/integration/container_workload/pool_image_test.py
+++ b/tests/integration/container_workload/pool_image_test.py
@@ -1,38 +1,70 @@
# stdlib
import os
-from time import sleep
# third party
from faker import Faker
import numpy as np
import pytest
+import requests
# syft absolute
import syft as sy
from syft.client.domain_client import DomainClient
from syft.custom_worker.config import DockerWorkerConfig
-from syft.node.node import get_default_worker_tag_by_env
from syft.service.request.request import Request
from syft.service.response import SyftSuccess
from syft.service.worker.worker_image import SyftWorkerImage
from syft.service.worker.worker_pool import SyftWorker
from syft.service.worker.worker_pool import WorkerPool
-SYFT_BASE_TAG = get_default_worker_tag_by_env()
-hagrid_flags = os.getenv("HAGRID_FLAGS")
-if hagrid_flags:
- SYFT_BASE_TAG = get_default_worker_tag_by_env(dev_mode=True)
+registry = os.getenv("SYFT_BASE_IMAGE_REGISTRY", "docker.io")
+repo = "openmined/grid-backend"
+
+if "k3d" in registry:
+ res = requests.get(url=f"http://{registry}/v2/{repo}/tags/list")
+ tag = res.json()["tags"][0]
+else:
+ tag = sy.__version__
+
+external_registry = os.getenv("EXTERNAL_REGISTRY", registry)
+external_registry_username = os.getenv("EXTERNAL_REGISTRY_USERNAME", None)
+external_registry_password = os.getenv("EXTERNAL_REGISTRY_PASSWORD", None)
+
+
+@pytest.fixture
+def external_registry_uid(domain_1_port):
+ domain_client: DomainClient = sy.login(
+ port=domain_1_port, email="info@openmined.org", password="changethis"
+ )
+ image_registry_list = domain_client.api.services.image_registry.get_all()
+ if len(image_registry_list) > 1:
+ raise Exception("Only one registry should be present for testing")
+
+ elif len(image_registry_list) == 1:
+ assert (
+ image_registry_list[0].url == external_registry
+ ), "External registry different from the one set in the environment variable"
+ return image_registry_list[0].id
+ else:
+ registry_add_result = domain_client.api.services.image_registry.add(
+ external_registry
+ )
+
+ assert isinstance(registry_add_result, sy.SyftSuccess), str(registry_add_result)
+
+ image_registry_list = domain_client.api.services.image_registry.get_all()
+ return image_registry_list[0].id
@pytest.mark.container_workload
-def test_image_build(domain_1_port) -> None:
+def test_image_build(domain_1_port, external_registry_uid) -> None:
domain_client: DomainClient = sy.login(
port=domain_1_port, email="info@openmined.org", password="changethis"
)
- # Submit Docker Worker Config
+ # Submit Docker Worker Config.
docker_config_rl = f"""
- FROM openmined/grid-backend:{SYFT_BASE_TAG}
+ FROM {registry}/{repo}:{tag}
RUN pip install recordlinkage
"""
docker_config = DockerWorkerConfig(dockerfile=docker_config_rl)
@@ -49,12 +81,11 @@ def test_image_build(domain_1_port) -> None:
assert not isinstance(workerimage, sy.SyftError)
# Build docker image
- tag_version = sy.UID().short()
- docker_tag = f"openmined/custom-worker-rl:{tag_version}"
+ docker_tag = "openmined/custom-worker-rl:latest"
docker_build_result = domain_client.api.services.worker_image.build(
image_uid=workerimage.id,
tag=docker_tag,
- pull=False,
+ registry_uid=external_registry_uid,
)
assert isinstance(docker_build_result, SyftSuccess)
@@ -67,18 +98,9 @@ def test_image_build(domain_1_port) -> None:
assert workerimage.image_identifier.repo_with_tag == docker_tag
assert workerimage.image_hash is not None
- # Delete image
- delete_result = domain_client.api.services.worker_image.remove(uid=workerimage.id)
- assert isinstance(delete_result, sy.SyftSuccess)
-
- # Validate the image is successfully deleted
- assert len(domain_client.images.get_all()) == 1
- workerimage = domain_client.images.get_all()[0]
- assert workerimage.config != docker_config
-
@pytest.mark.container_workload
-def test_pool_launch(domain_1_port) -> None:
+def test_pool_launch(domain_1_port, external_registry_uid) -> None:
domain_client: DomainClient = sy.login(
port=domain_1_port, email="info@openmined.org", password="changethis"
)
@@ -86,7 +108,7 @@ def test_pool_launch(domain_1_port) -> None:
# Submit Docker Worker Config
docker_config_opendp = f"""
- FROM openmined/grid-backend:{SYFT_BASE_TAG}
+ FROM {registry}/{repo}:{tag}
RUN pip install opendp
"""
docker_config = DockerWorkerConfig(dockerfile=docker_config_opendp)
@@ -103,18 +125,24 @@ def test_pool_launch(domain_1_port) -> None:
assert not worker_image.is_built
# Build docker image
- tag_version = sy.UID().short()
- docker_tag = f"openmined/custom-worker-opendp:{tag_version}"
+ docker_tag = "openmined/custom-worker-opendp:latest"
docker_build_result = domain_client.api.services.worker_image.build(
image_uid=worker_image.id,
tag=docker_tag,
- pull=False,
+ registry_uid=external_registry_uid,
)
assert isinstance(docker_build_result, SyftSuccess)
+ # Push Image to External registry
+ push_result = domain_client.api.services.worker_image.push(
+ worker_image.id,
+ username=external_registry_username,
+ password=external_registry_password,
+ )
+ assert isinstance(push_result, sy.SyftSuccess), str(push_result)
+
# Launch a worker pool
- pool_version = sy.UID().short()
- worker_pool_name = f"custom_worker_pool_ver{pool_version}"
+ worker_pool_name = "custom-worker-pool-opendp"
worker_pool_res = domain_client.api.services.worker_pool.launch(
name=worker_pool_name,
image_uid=worker_image.id,
@@ -156,14 +184,9 @@ def test_pool_launch(domain_1_port) -> None:
# TODO: delete the launched pool
- # Clean the build images
- sleep(10)
- delete_result = domain_client.api.services.worker_image.remove(uid=worker_image.id)
- assert isinstance(delete_result, sy.SyftSuccess)
-
@pytest.mark.container_workload
-def test_pool_image_creation_job_requests(domain_1_port) -> None:
+def test_pool_image_creation_job_requests(domain_1_port, external_registry_uid) -> None:
"""
Test register ds client, ds requests to create an image and pool creation,
do approves, then ds creates a function attached to the worker pool, then creates another
@@ -187,21 +210,19 @@ def test_pool_image_creation_job_requests(domain_1_port) -> None:
# the DS makes a request to create an image and a pool based on the image
docker_config_np = f"""
- FROM openmined/grid-backend:{SYFT_BASE_TAG}
+ FROM {registry}/{repo}:{tag}
RUN pip install numpy
"""
docker_config = DockerWorkerConfig(dockerfile=docker_config_np)
- tag_version = sy.UID().short()
- docker_tag = f"openmined/custom-worker-np:{tag_version}"
- pool_version = sy.UID().short()
- worker_pool_name = f"custom_worker_pool_ver{pool_version}"
+ docker_tag = "openmined/custom-worker-np:latest"
+ worker_pool_name = "custom-worker-pool-numpy"
request = ds_client.api.services.worker_pool.create_image_and_pool_request(
pool_name=worker_pool_name,
num_workers=1,
tag=docker_tag,
config=docker_config,
reason="I want to do some more cool data science with PySyft and Recordlinkage",
- pull_image=False,
+ registry_uid=external_registry_uid,
)
assert isinstance(request, Request)
assert len(request.changes) == 2
@@ -224,7 +245,7 @@ def test_pool_image_creation_job_requests(domain_1_port) -> None:
worker: SyftWorker = launched_pool.workers[0]
assert launched_pool.name in worker.name
- assert worker.status.value == "Pending"
+ assert worker.status.value == "Running"
assert worker.healthcheck.value == "β
"
# assert worker.consumer_state.value == "Idle"
assert isinstance(worker.logs, str)
@@ -279,8 +300,3 @@ def custom_worker_func(x):
assert isinstance(res, sy.SyftSuccess)
# TODO: delete the launched pool
-
- # Clean the build images
- sleep(10)
- delete_result = domain_client.api.services.worker_image.remove(uid=built_image.id)
- assert isinstance(delete_result, sy.SyftSuccess)
diff --git a/tests/integration/local/request_multiple_nodes_test.py b/tests/integration/local/request_multiple_nodes_test.py
index 601988673dc..e81f75b57d6 100644
--- a/tests/integration/local/request_multiple_nodes_test.py
+++ b/tests/integration/local/request_multiple_nodes_test.py
@@ -21,7 +21,6 @@ def node_1():
local_db=True,
create_producer=True,
n_consumers=1,
- in_memory_workers=True,
queue_port=None,
)
yield node
@@ -39,7 +38,6 @@ def node_2():
local_db=True,
create_producer=True,
n_consumers=1,
- in_memory_workers=True,
queue_port=None,
)
yield node
diff --git a/tests/integration/local/syft_function_test.py b/tests/integration/local/syft_function_test.py
index 6ca60f3b90d..8cc85cce4e2 100644
--- a/tests/integration/local/syft_function_test.py
+++ b/tests/integration/local/syft_function_test.py
@@ -23,7 +23,6 @@ def node():
n_consumers=3,
create_producer=True,
queue_port=None,
- in_memory_workers=True,
local_db=False,
)
# startup code here
diff --git a/tests/integration/local/twin_api_sync_test.py b/tests/integration/local/twin_api_sync_test.py
index d39066ade9a..e09c82001d1 100644
--- a/tests/integration/local/twin_api_sync_test.py
+++ b/tests/integration/local/twin_api_sync_test.py
@@ -111,7 +111,7 @@ def compute(query):
# verify that ds cannot access private job
assert client_low_ds.api.services.job.get(private_job_id) is None
- assert low_client.api.services.job.get(private_job_id) is not None
+ assert low_client.api.services.job.get(private_job_id) is None
# we only sync the mock function, we never sync the private function to the low side
mock_res = low_client.api.services.testapi.query.mock()
diff --git a/tox.ini b/tox.ini
index 41e6f168c95..c79fa513cb1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,7 +8,6 @@ envlist =
dev.k8s.cleanup
dev.k8s.destroy
dev.k8s.destroyall
- hagrid.publish
lint
stack.test.integration
syft.docs
@@ -69,16 +68,6 @@ allowlist_externals =
commands =
bash -c 'uv pip list || pip list'
-[testenv:hagrid]
-deps =
- -e{toxinidir}/packages/hagrid[dev]
-changedir = {toxinidir}/packages/hagrid
-description = Syft
-allowlist_externals =
- bash
-commands =
- bash -c 'uv pip list || pip list'
-
[testenv:syftcli]
deps =
-e{toxinidir}/packages/syftcli[dev]
@@ -98,14 +87,6 @@ commands =
python -c 'from shutil import rmtree; rmtree("build", True); rmtree("dist", True)'
python -m build .
-[testenv:hagrid.publish]
-changedir = {toxinidir}/packages/hagrid
-description = Build and Publish Hagrid Wheel
-deps =
- build
-commands =
- python -c 'from shutil import rmtree; rmtree("build", True); rmtree("dist", True)'
- python -m build .
[testenv:syftcli.publish]
changedir = {toxinidir}/packages/syftcli
@@ -179,8 +160,6 @@ commands =
[testenv:frontend.test.unit]
description = Frontend Unit Tests
-deps =
- {[testenv:hagrid]deps}
allowlist_externals =
docker
bash
@@ -201,145 +180,6 @@ commands =
docker run -t ui-test; \
fi'
-[testenv:frontend.test.e2e]
-description = Frontend Unit Tests
-deps =
- {[testenv:hagrid]deps}
-allowlist_externals =
- docker
- bash
- pnpm
- sleep
-passenv=HOME, USER
-changedir = {toxinidir}/packages/grid/frontend
-setenv =
- HAGRID_FLAGS = {env:HAGRID_FLAGS:--tag=local --test}
- ENABLE_SIGNUP=True
-commands =
- bash ./scripts/check_pnpm.sh
-
- bash -c "echo Running with HAGRID_FLAGS=$HAGRID_FLAGS; date"
-
- ; install hagrid
- bash -c 'if [[ "$HAGRID_FLAGS" == *"local"* ]]; then \
- uv pip install -e "../../hagrid"; \
- else \
- uv pip install --force hagrid; \
- fi'
-
- ; fix windows encoding
- - chcp 65001
-
- ; check docker versions
- bash -c "docker --version"
- bash -c "docker compose version"
-
- ; reset volumes and create nodes
- bash -c "echo Starting Nodes; date"
- bash -c "docker rm -f $(docker ps -a -q) || true"
- bash -c 'docker volume rm -f $(docker volume ls -q --filter "label=orgs.openmined.syft") || true'
-
- bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test_domain_1 domain to docker:9081 $HAGRID_FLAGS --enable-signup --no-health-checks --verbose --no-warnings'
-
- bash -c '(docker logs test_domain_1-frontend-1 -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'
- bash -c '(docker logs test_domain_1-backend-1 -f &) | grep -q "Application startup complete" || true'
-
- pnpm install
- pnpm dlx playwright@1.36.1 install --with-deps
- pnpm exec playwright install
- pnpm test:e2e
-
- ; shutdown
- bash -c "echo Killing Nodes; date"
- bash -c 'HAGRID_ART=false hagrid land all --force'
- bash -c 'docker volume rm -f $(docker volume ls -q --filter "label=orgs.openmined.syft") || true'
-
-
-[testenv:stack.test.integration]
-description = Integration Tests for Core Stack
-deps =
- {[testenv:syft]deps}
- {[testenv:hagrid]deps}
- pytest
-changedir = {toxinidir}
-allowlist_externals =
- docker
- grep
- sleep
- bash
- chcp
-passenv=HOME, USER, AZURE_BLOB_STORAGE_KEY
-setenv =
- HAGRID_FLAGS = {env:HAGRID_FLAGS:--tag=local --dev}
- EMULATION = {env:EMULATION:false}
- HAGRID_ART = false
- PYTHONIOENCODING = utf-8
- PYTEST_MODULES = {env:PYTEST_MODULES:frontend container_workload network local}
-commands =
- bash -c "whoami; id;"
-
- bash -c "echo Running with HAGRID_FLAGS=$HAGRID_FLAGS EMULATION=$EMULATION PYTEST_MODULES=$PYTEST_MODULES; date"
-
- ; install syft and hagrid
- bash -c 'if [[ "$HAGRID_FLAGS" == *"latest"* ]]; then \
- echo "Installing latest syft and hagrid"; \
- uv pip install --force hagrid syft; \
- elif [[ "$HAGRID_FLAGS" == *"beta"* ]]; then \
- echo "Installing beta syft and hagrid"; \
- uv pip install --force hagrid; \
- uv pip install --force -U --pre syft; \
- else \
- echo "Using local syft and hagrid"; \
- fi'
-
- ; fix windows encoding
- - chcp 65001
-
- ; check docker versions
- bash -c "docker --version"
- bash -c "docker compose version"
-
- ; reset volumes and create nodes
- bash -c "echo Starting Nodes; date"
- bash -c 'docker rm -f $(docker ps -a -q --filter "label=orgs.openmined.syft") || true'
- bash -c 'docker volume rm -f $(docker volume ls -q --filter "label=orgs.openmined.syft") || true'
-
- python -c 'import syft as sy; sy.stage_protocol_changes()'
-
- ; Make sure that pacakge-cache is owned by the current user
- ; instead of docker creating it as root
- bash -c 'mkdir -p packages/grid/data/package-cache'
-
- bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-gateway-1 gateway to docker:9081 $HAGRID_FLAGS --no-health-checks --verbose --no-warnings --build'
- bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-1 domain to docker:9082 $HAGRID_FLAGS --no-health-checks --enable-signup --verbose --no-warnings --build'
- ; bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-2 domain to docker:9083 --headless $HAGRID_FLAGS --enable-signup --no-health-checks --verbose --no-warnings --build'
-
- ; wait for nodes to start
- docker ps
- bash -c "echo Waiting for Nodes; date"
- bash -c '(docker logs test-domain-1-frontend-1 -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'
- bash -c '(docker logs test-domain-1-backend-1 -f &) | grep -q "Application startup complete" || true'
- ; bash -c '(docker logs test_domain_2-backend-1 -f &) | grep -q "Application startup complete" || true'
- bash -c '(docker logs test-gateway-1-backend-1 -f &) | grep -q "Application startup complete" || true'
-
- bash -c '\
- PYTEST_MODULES=($PYTEST_MODULES); \
- for i in "${PYTEST_MODULES[@]}"; do \
- echo "Starting test for $i"; date; \
- pytest tests/integration -m $i -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
- return=$?; \
- echo "Finished $i"; \
- date; \
- if [[ $return -ne 0 ]]; then \
- exit $return; \
- fi; \
- done'
-
- ; shutdown
- bash -c "echo Killing Nodes; date"
- bash -c 'HAGRID_ART=false hagrid land all --force'
- bash -c 'docker rm -f $(docker ps -a -q --filter "label=orgs.openmined.syft") || true'
- bash -c 'docker volume rm -f $(docker volume ls -q --filter "label=orgs.openmined.syft") || true'
[testenv:syft.docs]
@@ -347,7 +187,6 @@ description = Build Docs for Syft
changedir = {toxinidir}/docs
deps =
{[testenv:syft]deps}
- {[testenv:hagrid]deps}
-r {toxinidir}/docs/requirements.txt
allowlist_externals =
make
@@ -368,7 +207,6 @@ commands =
description = Jupyter Notebook with Editable Syft
deps =
{[testenv:syft]deps}
- {[testenv:hagrid]deps}
jupyter
jupyterlab
commands =
@@ -395,7 +233,6 @@ description = Security Checks for Syft
changedir = {toxinidir}/packages/syft
deps =
{[testenv:syft]deps}
- {[testenv:hagrid]deps}
commands =
bandit -r src
# ansible 8.4.0
@@ -406,7 +243,6 @@ commands =
description = Syft Unit Tests
deps =
{[testenv:syft]deps}
- {[testenv:hagrid]deps}
allowlist_externals =
bash
uv
@@ -421,7 +257,6 @@ commands =
description = Syft Notebook Tests
deps =
-e{toxinidir}/packages/syft[dev,data_science]
- {[testenv:hagrid]deps}
nbmake
changedir = {toxinidir}/notebooks
allowlist_externals =
@@ -490,20 +325,18 @@ commands =
description = Stack Notebook Tests
deps =
{[testenv:syft]deps}
- {[testenv:hagrid]deps}
nbmake
changedir = {toxinidir}/notebooks
allowlist_externals =
bash
setenv =
- ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s}
+ ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:remote}
DEV_MODE = {env:DEV_MODE:True}
TEST_NOTEBOOK_PATHS = {env:TEST_NOTEBOOK_PATHS:api/0.8}
ENABLE_SIGNUP=True
commands =
# Volume cleanup
- bash -c 'hagrid land all --force --prune-vol || true'
bash -c 'docker volume rm -f $(docker volume ls -q --filter "label=orgs.openmined.syft") || true'
bash -c 'docker volume rm -f $(docker volume ls -q --filter "label=com.docker.volume.anonymous") || true'
bash -c 'docker network rm -f $(docker network ls -q --filter "label=orgs.openmined.syft") || true'
@@ -519,7 +352,6 @@ commands =
; pytest -x --nbmake --nbmake-timeout=1000 tutorials -p no:randomly -vvvv
; pytest -x --nbmake --nbmake-timeout=1000 tutorials/pandas-cookbook -p no:randomly -vvvv
- bash -c 'hagrid land all --force --prune-vol'
bash -c 'docker volume rm -f $(docker volume ls -q --filter "label=orgs.openmined.syft") || true'
bash -c 'docker volume rm -f $(docker volume ls -q --filter "label=com.docker.volume.anonymous") || true'
bash -c 'docker network rm -f $(docker network ls -q --filter "label=orgs.openmined.syft") || true'
@@ -528,7 +360,6 @@ commands =
description = Stack VM Tests
deps =
{[testenv:syft]deps}
- {[testenv:hagrid]deps}
nbmake
allowlist_externals =
cd
@@ -578,7 +409,6 @@ commands =
description = Stack podman Tests for Rhel & Centos
deps =
{[testenv:syft]deps}
- {[testenv:hagrid]deps}
nbmake
allowlist_externals =
cd
@@ -611,7 +441,6 @@ commands =
description = Generate Types for Frontend
deps =
{[testenv:syft]deps}
- {[testenv:hagrid]deps}
allowlist_externals =
cd
bash
@@ -636,16 +465,43 @@ commands =
python_version = 3.12
disable_error_code = attr-defined, valid-type, no-untyped-call, arg-type
+[testenv:syft.test.integration]
+description = Integration Tests for Syft Stack
+basepython = python3
+deps =
+ {[testenv:syft]deps}
+changedir = {toxinidir}
+passenv=HOME, USER
+allowlist_externals =
+ bash
+setenv =
+ PYTEST_MODULES = {env:PYTEST_MODULES:local_node}
+ ASSOCIATION_REQUEST_AUTO_APPROVAL = {env:ASSOCIATION_REQUEST_AUTO_APPROVAL:true}
+ PYTEST_FLAGS = {env:PYTEST_FLAGS:--ignore=tests/integration/local/gateway_local_test.py --ignore=tests/integration/local/job_test.py}
+commands =
+ python -c 'import syft as sy; sy.stage_protocol_changes()'
+
+ # Run Integration Tests
+ bash -c '\
+ PYTEST_MODULES=($PYTEST_MODULES); \
+ for i in "${PYTEST_MODULES[@]}"; do \
+ echo "Starting test for $i"; date; \
+ pytest tests/integration -m $i -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no $PYTEST_FLAGS; \
+ return=$?; \
+ echo "Finished $i"; \
+ date; \
+ if [[ $return -ne 0 ]]; then \
+ exit $return; \
+ fi; \
+ done'
[testenv:stack.test.integration.k8s]
-description = Integration Tests for Core Stack
+description = Integration Tests for Core Stack using K8s
basepython = python3
deps =
{[testenv:syft]deps}
- {[testenv:hagrid]deps}
- nbmake
changedir = {toxinidir}
-passenv=HOME, USER
+passenv=HOME, USER, AZURE_BLOB_STORAGE_KEY
allowlist_externals =
devspace
kubectl
@@ -657,41 +513,39 @@ allowlist_externals =
echo
tox
setenv =
- ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s}
NODE_PORT = {env:NODE_PORT:9082}
GITHUB_CI = {env:GITHUB_CI:false}
- PYTEST_MODULES = {env:PYTEST_MODULES:frontend container_workload local}
- SYFT_BASE_IMAGE_REGISTRY = {env:SYFT_BASE_IMAGE_REGISTRY:k3d-registry.localhost:5800}
+ PYTEST_MODULES = {env:PYTEST_MODULES:frontend network container_workload}
+ DOMAIN_CLUSTER_NAME = {env:DOMAIN_CLUSTER_NAME:test-domain-1}
+ GATEWAY_CLUSTER_NAME = {env:GATEWAY_CLUSTER_NAME:test-gateway-1}
ASSOCIATION_REQUEST_AUTO_APPROVAL = {env:ASSOCIATION_REQUEST_AUTO_APPROVAL:true}
+ SYFT_BASE_IMAGE_REGISTRY = {env:SYFT_BASE_IMAGE_REGISTRY:k3d-registry.localhost:5800}
commands =
bash -c "echo Running with GITHUB_CI=$GITHUB_CI; date"
python -c 'import syft as sy; sy.stage_protocol_changes()'
k3d version
- # Since cluster name cannot have underscore and environment variable cannot have hyphen
- # we are passing a grouped name for node names
- # bash -c "docker rm $(docker ps -aq) --force || true"
- # Deleting current cluster
- bash -c "k3d cluster delete testgateway1 || true"
- bash -c "k3d cluster delete testdomain1 || true"
+ # Deleting Old Cluster
+ bash -c "k3d cluster delete ${DOMAIN_CLUSTER_NAME} || true"
+ bash -c "k3d cluster delete ${GATEWAY_CLUSTER_NAME} || true"
# Deleting registry & volumes
bash -c "k3d registry delete k3d-registry.localhost || true"
- bash -c "docker volume rm k3d-testgateway1-images --force || true"
- bash -c "docker volume rm k3d-testdomain1-images --force || true"
+ bash -c "docker volume rm k3d-${DOMAIN_CLUSTER_NAME}-images --force || true"
+ bash -c "docker volume rm k3d-${GATEWAY_CLUSTER_NAME}-images --force || true"
# Create registry
tox -e dev.k8s.registry
- # Creating testgateway1 cluster on port 9081
+ # Creating test-gateway-1 cluster on port 9081
bash -c '\
- export CLUSTER_NAME=testgateway1 CLUSTER_HTTP_PORT=9081 DEVSPACE_PROFILE=gateway && \
+ export CLUSTER_NAME=${GATEWAY_CLUSTER_NAME} CLUSTER_HTTP_PORT=9081 DEVSPACE_PROFILE=gateway && \
tox -e dev.k8s.start && \
tox -e dev.k8s.deploy'
- # Creating testdomain1 cluster on port 9082
+ # Creating test-domain-1 cluster on port 9082
bash -c '\
- export CLUSTER_NAME=testdomain1 CLUSTER_HTTP_PORT=9082 && \
+ export CLUSTER_NAME=${DOMAIN_CLUSTER_NAME} CLUSTER_HTTP_PORT=9082 && \
tox -e dev.k8s.start && \
tox -e dev.k8s.deploy'
@@ -703,65 +557,115 @@ commands =
sleep 30
- # wait for front end
- bash packages/grid/scripts/wait_for.sh service frontend --context k3d-testdomain1 --namespace syft
- bash -c '(kubectl logs service/frontend --context k3d-testdomain1 --namespace syft -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'
-
# wait for test gateway 1
- bash packages/grid/scripts/wait_for.sh service mongo --context k3d-testgateway1 --namespace syft
- bash packages/grid/scripts/wait_for.sh service backend --context k3d-testgateway1 --namespace syft
- bash packages/grid/scripts/wait_for.sh service proxy --context k3d-testgateway1 --namespace syft
+ bash packages/grid/scripts/wait_for.sh service mongo --context k3d-{env:GATEWAY_CLUSTER_NAME} --namespace syft
+ bash packages/grid/scripts/wait_for.sh service backend --context k3d-{env:GATEWAY_CLUSTER_NAME} --namespace syft
+ bash packages/grid/scripts/wait_for.sh service proxy --context k3d-{env:GATEWAY_CLUSTER_NAME} --namespace syft
# wait for test domain 1
- bash packages/grid/scripts/wait_for.sh service mongo --context k3d-testdomain1 --namespace syft
- bash packages/grid/scripts/wait_for.sh service backend --context k3d-testdomain1 --namespace syft
- bash packages/grid/scripts/wait_for.sh service proxy --context k3d-testdomain1 --namespace syft
- bash packages/grid/scripts/wait_for.sh service seaweedfs --context k3d-testdomain1 --namespace syft
+ bash packages/grid/scripts/wait_for.sh service mongo --context k3d-{env:DOMAIN_CLUSTER_NAME} --namespace syft
+ bash packages/grid/scripts/wait_for.sh service backend --context k3d-{env:DOMAIN_CLUSTER_NAME} --namespace syft
+ bash packages/grid/scripts/wait_for.sh service proxy --context k3d-{env:DOMAIN_CLUSTER_NAME} --namespace syft
+ bash packages/grid/scripts/wait_for.sh service seaweedfs --context k3d-{env:DOMAIN_CLUSTER_NAME} --namespace syft
+ bash packages/grid/scripts/wait_for.sh service frontend --context k3d-{env:DOMAIN_CLUSTER_NAME} --namespace syft
+ bash -c '(kubectl logs service/frontend --context k3d-${DOMAIN_CLUSTER_NAME} --namespace syft -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'
# Checking logs generated & startup of test-domain 1
- bash -c '(kubectl logs service/backend --context k3d-testdomain1 --namespace syft -f &) | grep -q "Application startup complete" || true'
+ bash -c '(kubectl logs service/backend --context k3d-${DOMAIN_CLUSTER_NAME} --namespace syft -f &) | grep -q "Application startup complete" || true'
# Checking logs generated & startup of testgateway1
- bash -c '(kubectl logs service/backend --context k3d-testgateway1 --namespace syft -f &) | grep -q "Application startup complete" || true'
-
- # frontend
- bash -c 'if [[ "$PYTEST_MODULES" == *"frontend"* ]]; then \
- echo "Starting frontend"; date; \
- pytest tests/integration -m frontend -p no:randomly -k "test_serves_domain_frontend" --co; \
- pytest tests/integration -m frontend -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no -k "test_serves_domain_frontend"; \
- return=$?; \
- echo "Finished frontend"; date; \
- exit $return; \
+ bash -c '(kubectl logs service/backend --context k3d-${GATEWAY_CLUSTER_NAME} --namespace syft -f &) | grep -q "Application startup complete" || true'
+
+ # Run Integration Tests
+ bash -c '\
+ PYTEST_MODULES=($PYTEST_MODULES); \
+ for i in "${PYTEST_MODULES[@]}"; do \
+ echo "Starting test for $i"; date; \
+ pytest tests/integration -m $i -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
+ return=$?; \
+ echo "Finished $i"; \
+ date; \
+ if [[ $return -ne 0 ]]; then \
+ exit $return; \
+ fi; \
+ done'
+
+ # deleting clusters created
+ bash -c "CLUSTER_NAME=${DOMAIN_CLUSTER_NAME} tox -e dev.k8s.destroy || true"
+ bash -c "CLUSTER_NAME=${GATEWAY_CLUSTER_NAME} tox -e dev.k8s.destroy || true"
+ bash -c "k3d registry delete k3d-registry.localhost || true"
+ bash -c "docker volume rm k3d-${DOMAIN_CLUSTER_NAME}-images --force || true"
+ bash -c "docker volume rm k3d-${GATEWAY_CLUSTER_NAME}-images --force || true"
+
+[testenv:stack.test.notebook.k8s]
+description = Notebook Tests for Core Stack using K8s
+basepython = python3
+deps =
+ {[testenv:syft]deps}
+ nbmake
+changedir = {toxinidir}
+passenv=HOME, USER
+allowlist_externals =
+ devspace
+ kubectl
+ grep
+ sleep
+ bash
+ k3d
+ echo
+ tox
+setenv =
+ ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:remote}
+ GITHUB_CI = {env:GITHUB_CI:false}
+ SYFT_BASE_IMAGE_REGISTRY = {env:SYFT_BASE_IMAGE_REGISTRY:k3d-registry.localhost:5800}
+ DOMAIN_CLUSTER_NAME = {env:DOMAIN_CLUSTER_NAME:test-domain-1}
+ NODE_PORT = {env:NODE_PORT:8080}
+commands =
+ bash -c "echo Running with GITHUB_CI=$GITHUB_CI; date"
+ python -c 'import syft as sy; sy.stage_protocol_changes()'
+ k3d version
+
+ # Deleting Old Cluster
+ bash -c "k3d cluster delete ${DOMAIN_CLUSTER_NAME} || true"
+
+ # Deleting registry & volumes
+ bash -c "k3d registry delete k3d-registry.localhost || true"
+ bash -c "docker volume rm k3d-${DOMAIN_CLUSTER_NAME}-images --force || true"
+
+ # Create registry
+ tox -e dev.k8s.registry
+
+
+ # Creating test-domain-1 cluster on port NODE_PORT
+ bash -c '\
+ export CLUSTER_NAME=${DOMAIN_CLUSTER_NAME} CLUSTER_HTTP_PORT=${NODE_PORT} && \
+ tox -e dev.k8s.start && \
+ tox -e dev.k8s.deploy'
+
+ # free up build cache after build of images
+ bash -c 'if [[ "$GITHUB_CI" != "false" ]]; then \
+ docker image prune --all --force; \
+ docker builder prune --all --force; \
fi'
- # Integration + Gateway Connection Tests
- # Gateway tests are not run in kuberetes, as currently,it does not have a way to configure
- # high/low side warning flag.
- bash -c "source ./scripts/get_k8s_secret_ci.sh; \
- pytest tests/integration/network -k 'not test_domain_gateway_user_code' -p no:randomly -vvvv"
-
- # Shutting down the gateway cluster to free up space, as the
- # below code does not require gateway cluster
- bash -c "CLUSTER_NAME=testgateway1 tox -e dev.k8s.destroy || true"
- bash -c "docker volume rm k3d-testgateway1-images --force || true"
-
- ; container workload
- ; bash -c 'if [[ "$PYTEST_MODULES" == *"container_workload"* ]]; then \
- ; echo "Starting Container Workload test"; date; \
- ; pytest tests/integration -m container_workload -p no:randomly --co; \
- ; pytest tests/integration -m container_workload -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
- ; return=$?; \
- ; echo "Finished container workload"; date; \
- ; exit $return; \
- ; fi'
-
- bash -c "source ./scripts/get_k8s_secret_ci.sh; \
- pytest -x --nbmake --nbmake-timeout=1000 notebooks/api/0.8 -p no:randomly -k 'not 10-container-images.ipynb' -vvvv"
+ sleep 30
+
+ # wait for test-domain-1
+ bash packages/grid/scripts/wait_for.sh service mongo --context k3d-{env:DOMAIN_CLUSTER_NAME} --namespace syft
+ bash packages/grid/scripts/wait_for.sh service backend --context k3d-{env:DOMAIN_CLUSTER_NAME} --namespace syft
+ bash packages/grid/scripts/wait_for.sh service proxy --context k3d-{env:DOMAIN_CLUSTER_NAME} --namespace syft
+ bash packages/grid/scripts/wait_for.sh service seaweedfs --context k3d-{env:DOMAIN_CLUSTER_NAME} --namespace syft
+ bash packages/grid/scripts/wait_for.sh service frontend --context k3d-{env:DOMAIN_CLUSTER_NAME} --namespace syft
+ bash -c '(kubectl logs service/frontend --context k3d-${DOMAIN_CLUSTER_NAME} --namespace syft -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'
+
+ # Checking logs generated & startup of test-domain 1
+ bash -c '(kubectl logs service/backend --context k3d-${DOMAIN_CLUSTER_NAME} --namespace syft -f &) | grep -q "Application startup complete" || true'
+
+ bash -c "pytest -x --nbmake notebooks/api/0.8 -p no:randomly -k 'not 10-container-images.ipynb' -vvvv --nbmake-timeout=1000"
# deleting clusters created
- bash -c "CLUSTER_NAME=testdomain1 tox -e dev.k8s.destroy || true"
+ bash -c "CLUSTER_NAME=${DOMAIN_CLUSTER_NAME} tox -e dev.k8s.destroy || true"
bash -c "k3d registry delete k3d-registry.localhost || true"
- bash -c "docker rm $(docker ps -aq) --force || true"
- bash -c "docker volume rm k3d-testdomain1-images --force || true"
+ bash -c "docker volume rm k3d-${DOMAIN_CLUSTER_NAME}-images --force || true"
[testenv:syft.build.helm]
@@ -817,7 +721,7 @@ allowlist_externals =
bash
tox
setenv =
- ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s}
+ ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:remote}
NODE_PORT = {env:NODE_PORT:8080}
NODE_URL = {env:NODE_URL:http://localhost}
EXCLUDE_NOTEBOOKS = {env:EXCLUDE_NOTEBOOKS:not 10-container-images.ipynb}
@@ -890,7 +794,8 @@ commands =
bash -c 'k3d --version'
; create registry
- bash -c 'k3d registry create registry.localhost --port 5800 -v $HOME/.k3d-registry:/var/lib/registry || true'
+ bash -c 'docker volume create k3d-registry-vol || true'
+ bash -c 'k3d registry create registry.localhost --port 5800 -v k3d-registry-vol:/var/lib/registry --no-help || true'
; add patches to host
bash -c 'if ! grep -q k3d-registry.localhost /etc/hosts; then sudo {envpython} scripts/patch_hosts.py --add-k3d-registry --fix-docker-hosts; fi'
@@ -1018,7 +923,7 @@ commands =
tox -e dev.k8s.{posargs:deploy}
[testenv:dev.k8s.launch.enclave]
-description = Launch a single Enclave on K8s
+description = Launch a single Enclave on K8s
passenv = HOME, USER
setenv=
CLUSTER_NAME = {env:CLUSTER_NAME:test-enclave-1}
@@ -1040,9 +945,6 @@ allowlist_externals =
tox
bash
commands =
- ; purge deployment and dangling resources
- tox -e dev.k8s.cleanup
-
; destroy cluster
bash -c '\
rm -rf .devspace; echo ""; \
@@ -1062,7 +964,7 @@ commands =
; destroy registry
bash -c 'k3d registry delete registry.localhost || true'
- bash -c 'sudo rm -rf ~/.k3d-registry'
+ bash -c 'docker volume rm k3d-registry-vol --force || true'
[testenv:backend.test.basecpu]
description = Base CPU Docker Image Test
@@ -1123,7 +1025,7 @@ allowlist_externals =
pytest
passenv = EXTERNAL_REGISTRY,EXTERNAL_REGISTRY_USERNAME,EXTERNAL_REGISTRY_PASSWORD
setenv =
- ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s}
+ ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:remote}
NODE_PORT = {env:NODE_PORT:8080}
NODE_URL = {env:NODE_URL:http://localhost}
EXCLUDE_NOTEBOOKS = {env:EXCLUDE_NOTEBOOKS:}