Skip to content

Commit

Permalink
Merge pull request #8398 from OpenMined/dsdo-image-requestapproval
Browse files Browse the repository at this point in the history 
DS/DO Flow for Request/Approve CustomImage
  • Loading branch information
@shubham3121
shubham3121 committed Jan 19, 2024
2 parents 306fdb3 + f20e933 commit b5ee778
Show file tree
Hide file tree
Showing 12 changed files with 824 additions and 14 deletions.
347 changes: 344 additions & 3 deletions notebooks/api/0.8/10-container-images.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
"\n",
"sy.requires(SYFT_VERSION)\n",
"# syft absolute\n",
"from syft.custom_worker.config import DockerWorkerConfig\n",
"from syft.service.worker.image_registry import SyftImageRegistry\n",
"from syft.service.worker.worker_image import SyftWorkerImage\n",
"\n",
Expand Down Expand Up @@ -70,6 +69,7 @@
" create_producer=True,\n",
" in_memory_workers=in_memory_workers,\n",
" reset=True,\n",
" port=8081,\n",
")"
]
},
Expand Down Expand Up @@ -131,7 +131,7 @@
"metadata": {},
"outputs": [],
"source": [
"docker_config = DockerWorkerConfig(dockerfile=custom_dockerfile_str)"
"docker_config = sy.DockerWorkerConfig(dockerfile=custom_dockerfile_str)"
]
},
{
Expand Down Expand Up @@ -357,6 +357,7 @@
"outputs": [],
"source": [
"docker_tag = \"openmined/custom-worker:0.7.8\"\n",
"\n",
"registry_uid = local_registry.id if running_as_container else local_registry.id\n",
"\n",
"docker_build_result = domain_client.api.services.worker_image.build(\n",
Expand Down Expand Up @@ -1029,6 +1030,346 @@
"if running_as_container:\n",
" local_registry_container.teardown()"
]
},
{
"cell_type": "markdown",
"id": "f20a29df-2e63-484f-8b67-d6a397722e66",
"metadata": {},
"source": [
"#### Worker Pool and Image Creation Request/Approval"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2b8cd7a0-ba17-4ad0-b3de-5af1282a6dc6",
"metadata": {},
"outputs": [],
"source": [
"custom_dockerfile_str_2 = \"\"\"\n",
"FROM openmined/grid-backend:0.8.4-beta.12\n",
"\n",
"RUN pip install opendp\n",
"\"\"\"\n",
"\n",
"docker_config_2 = sy.DockerWorkerConfig(dockerfile=custom_dockerfile_str_2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "48a7a9b5-266d-4f22-9b99-061dbb3c83ab",
"metadata": {},
"outputs": [],
"source": [
"submit_result = domain_client.api.services.worker_image.submit_dockerfile(\n",
" docker_config=docker_config_2\n",
")\n",
"submit_result"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4b3880fe-d682-471d-a52b-364711bf8511",
"metadata": {},
"outputs": [],
"source": [
"domain_client.images"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b62871bc-6c32-4fac-95af-5b062bc65992",
"metadata": {},
"outputs": [],
"source": [
"# get the image that's not built\n",
"workerimage_2 = None\n",
"for im in domain_client.images:\n",
" if im.config == docker_config_2:\n",
" workerimage_2 = im"
]
},
{
"cell_type": "markdown",
"id": "35f8e35f-91f3-4d2b-8e70-386021e9a692",
"metadata": {},
"source": [
"##### Build image first then create pool"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f5a773e7-4dc1-4325-bc26-eb3c7d88969a",
"metadata": {},
"outputs": [],
"source": [
"docker_tag_2 = \"openmined/custom-worker-opendp:latest\"\n",
"\n",
"docker_build_result = domain_client.api.services.worker_image.build(\n",
" image_uid=workerimage_2.id,\n",
" tag=docker_tag_2,\n",
")\n",
"docker_build_result"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7b0b2bb2-5612-463f-af88-f74e4f31719a",
"metadata": {},
"outputs": [],
"source": [
"pool_create_request = domain_client.api.services.worker_pool.pool_creation_request(\n",
" pool_name=\"first-opendp-pool\", num_workers=3, image_uid=workerimage_2.id\n",
")\n",
"pool_create_request"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2b337373-9486-426a-a282-b0b179139ba7",
"metadata": {},
"outputs": [],
"source": [
"assert len(pool_create_request.changes) == 1"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0b59e175-76ba-46b8-a7cd-796a872969e4",
"metadata": {},
"outputs": [],
"source": [
"# get the pending request and approve it\n",
"req_result = pool_create_request.approve()\n",
"req_result"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4ce90111-11bd-4ebd-bb4a-4217a57c7d8d",
"metadata": {},
"outputs": [],
"source": [
"assert isinstance(req_result, sy.SyftSuccess)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2ea69b17-eb3c-4f01-9a47-4895dd286e5e",
"metadata": {},
"outputs": [],
"source": [
"assert domain_client.worker_pools[\"first-opendp-pool\"]\n",
"assert len(domain_client.worker_pools[\"first-opendp-pool\"].worker_list) == 3"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b0f8e4cb-6ccf-4c9f-866e-6e63fa67427c",
"metadata": {},
"outputs": [],
"source": [
"assert len(domain_client.worker_pools.get_all()) == 3"
]
},
{
"cell_type": "markdown",
"id": "1340b532-f3bb-4afb-b777-9fb2ba4bd02c",
"metadata": {},
"source": [
"##### Request to build the image and create the pool at the same time"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8ead0843-d250-409f-a546-8049d9103646",
"metadata": {},
"outputs": [],
"source": [
"custom_dockerfile_str_3 = \"\"\"\n",
"FROM openmined/grid-backend:0.8.4-beta.12\n",
"\n",
"RUN pip install recordlinkage\n",
"\"\"\"\n",
"\n",
"docker_config_3 = sy.DockerWorkerConfig(dockerfile=custom_dockerfile_str_3)\n",
"\n",
"docker_tag_3 = \"openmined/custom-worker-recordlinkage:latest\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6732056f",
"metadata": {},
"outputs": [],
"source": [
"test_build_res = docker_config_3.test_image_build(tag=docker_tag_3)\n",
"assert isinstance(test_build_res, sy.SyftSuccess)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "441ff01a-6f0c-48db-a14d-deecb4518e18",
"metadata": {},
"outputs": [],
"source": [
"pool_image_create_request = (\n",
" domain_client.api.services.worker_pool.create_image_and_pool_request(\n",
" pool_name=\"recordlinkage-pool\",\n",
" num_workers=2,\n",
" tag=docker_tag_3,\n",
" config=docker_config_3,\n",
" reason=\"I want to do some more cool data science with PySyft and OpenDP\",\n",
" )\n",
")\n",
"pool_image_create_request"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3c1a1cf0-a31f-4dcc-bc34-8a232fb23b62",
"metadata": {},
"outputs": [],
"source": [
"assert len(pool_image_create_request.changes) == 2\n",
"assert pool_image_create_request.changes[0].config == docker_config_3\n",
"assert pool_image_create_request.changes[1].num_workers == 2\n",
"assert pool_image_create_request.changes[1].pool_name == \"recordlinkage-pool\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f456f727-ca38-4872-9789-e457f211ce6d",
"metadata": {},
"outputs": [],
"source": [
"# get the pending request and approve it\n",
"req_result = pool_image_create_request.approve()\n",
"req_result"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6d358265-a2eb-4791-84c4-0e2d0cc88f8a",
"metadata": {},
"outputs": [],
"source": [
"assert isinstance(req_result, sy.SyftSuccess)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "83188182-1e58-4d6b-a361-b9ab4fcea356",
"metadata": {},
"outputs": [],
"source": [
"# Get updated request object and status\n",
"for req in domain_client.requests:\n",
" if req.id == pool_image_create_request.id:\n",
" pool_image_create_request = req\n",
"\n",
"assert pool_image_create_request.status.value == 2"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6c6760aa-f26b-49b6-9346-416b8e1cca1a",
"metadata": {},
"outputs": [],
"source": [
"domain_client.images"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e3c26241-028b-4f6d-a9dc-c16250f3ac6c",
"metadata": {},
"outputs": [],
"source": [
"image_exists = False\n",
"for im in domain_client.images.get_all():\n",
" if im.image_identifier and im.image_identifier.repo_with_tag == docker_tag_3:\n",
" image_exists = True\n",
"assert image_exists"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7016eccb-8830-4d9f-b1f6-da3dbafeb0f8",
"metadata": {},
"outputs": [],
"source": [
"assert domain_client.worker_pools[\"recordlinkage-pool\"]\n",
"assert len(domain_client.worker_pools[\"recordlinkage-pool\"].worker_list) == 2"
]
},
{
"cell_type": "markdown",
"id": "ca0febe0-ab67-441a-92c2-f3de243bf940",
"metadata": {},
"source": [
"#### Clean up workers"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c0317e06-fd94-43d4-88d5-af39033aafe0",
"metadata": {},
"outputs": [],
"source": [
"# delete the remaining workers\n",
"for worker_pool in domain_client.worker_pools:\n",
" for worker in worker_pool.workers:\n",
" res = domain_client.api.services.worker.delete(uid=worker.id, force=True)\n",
" print(res)\n",
" assert isinstance(res, sy.SyftSuccess)\n",
"\n",
"# Adding some sleep to allow containers to be fully removed,\n",
"# before removing the image\n",
"time.sleep(10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2c809521-cb0d-432f-b75a-7da6d635e85d",
"metadata": {},
"outputs": [],
"source": [
"for worker_pool in domain_client.worker_pools:\n",
" assert len(worker_pool.worker_list) == 0"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6391a086-604a-47a9-959d-d4a626ac57f2",
"metadata": {},
"outputs": [],
"source": [
"domain.land()"
]
}
],
"metadata": {
Expand All @@ -1047,7 +1388,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.5"
"version": "3.9.7"
}
},
"nbformat": 4,
Expand Down
1 change: 1 addition & 0 deletions packages/syft/src/syft/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from .client.search import SearchResults # noqa: F401
from .client.user_settings import UserSettings # noqa: F401
from .client.user_settings import settings # noqa: F401
from .custom_worker.config import DockerWorkerConfig # noqa: F401
from .external import OBLV # noqa: F401
from .external import enable_external_lib # noqa: F401
from .node.credentials import SyftSigningKey # noqa: F401
Expand Down
Loading

0 comments on commit b5ee778

Please sign in to comment.