Skip to content

Commit

Permalink
Merge pull request #8582 from OpenMined/rasswanth/optimize-veilid
Browse files Browse the repository at this point in the history 
Veilid Optimization Iteration 2
  • Loading branch information
@rasswanth-s
rasswanth-s authored Mar 14, 2024
2 parents 2b2d9c2 + 9515dda commit 47c132c
Show file tree
Hide file tree
Showing 17 changed files with 297 additions and 225 deletions.
43 changes: 27 additions & 16 deletions notebooks/Testing/Veilid/Alice-Python-Server.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
"metadata": {},
"outputs": [],
"source": [
"res = requests.post(f\"http://{host}:{port}/generate_dht_key\")"
"res = requests.post(f\"http://{host}:{port}/generate_vld_key\")"
]
},
{
Expand All @@ -65,7 +65,7 @@
"metadata": {},
"outputs": [],
"source": [
"res = requests.get(f\"http://{host}:{port}/retrieve_dht_key\")"
"res = requests.get(f\"http://{host}:{port}/retrieve_vld_key\")"
]
},
{
Expand All @@ -75,9 +75,9 @@
"metadata": {},
"outputs": [],
"source": [
"self_dht_key = res.json()[\"message\"]\n",
"self_vld_key = res.json()[\"message\"]\n",
"print(\"=\" * 30)\n",
"print(self_dht_key)\n",
"print(self_vld_key)\n",
"print(\"=\" * 30)"
]
},
Expand All @@ -86,7 +86,18 @@
"id": "a8c70d99-6814-453d-80bf-d141c40ba24e",
"metadata": {},
"source": [
"### Send AppMessage using DHT Key to Self"
"### Send AppMessage using VLD Key to Self"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a7495805-817d-44d9-ad62-32407b42316c",
"metadata": {},
"outputs": [],
"source": [
"# Cannot send messages to self, due to local routing feature not\n",
"# available in direct routing"
]
},
{
Expand All @@ -96,8 +107,8 @@
"metadata": {},
"outputs": [],
"source": [
"json_data = {\"dht_key\": self_dht_key, \"message\": \"Hello to me again\"}\n",
"app_message = requests.post(f\"http://{host}:{port}/app_message\", json=json_data)"
"# json_data = {\"dht_key\": self_dht_key, \"message\": \"Hello to me again\"}\n",
"# app_message = requests.post(f\"http://{host}:{port}/app_message\", json=json_data)"
]
},
{
Expand All @@ -107,15 +118,15 @@
"metadata": {},
"outputs": [],
"source": [
"app_message.content"
"# app_message.content"
]
},
{
"cell_type": "markdown",
"id": "4d0d9e39-bf05-4ef3-b00a-2bb605f041ee",
"metadata": {},
"source": [
"### Send AppCall using DHT Key to Self"
"### Send AppCall using VLD Key to Self"
]
},
{
Expand All @@ -125,8 +136,8 @@
"metadata": {},
"outputs": [],
"source": [
"json_data = {\"dht_key\": self_dht_key, \"message\": \"Hello to app call\"}\n",
"app_call = requests.post(f\"http://{host}:{port}/app_call\", json=json_data)"
"# json_data = {\"dht_key\": self_dht_key, \"message\": \"Hello to app call\"}\n",
"# app_call = requests.post(f\"http://{host}:{port}/app_call\", json=json_data)"
]
},
{
Expand All @@ -136,15 +147,15 @@
"metadata": {},
"outputs": [],
"source": [
"app_call.json()"
"# app_call.json()"
]
},
{
"cell_type": "markdown",
"id": "fd824cca-2a7f-4ea9-9e67-1c06d1f8bec2",
"metadata": {},
"source": [
"### Send AppMessage using DHT Key to Peer"
"### Send AppMessage using VLD Key to Peer"
]
},
{
Expand All @@ -154,7 +165,7 @@
"metadata": {},
"outputs": [],
"source": [
"peer_dht_key = input(\"Enter Peer DHT Key\")"
"peer_vld_key = input(\"Enter Peer VLD Key\")"
]
},
{
Expand All @@ -164,7 +175,7 @@
"metadata": {},
"outputs": [],
"source": [
"json_data = {\"dht_key\": peer_dht_key, \"message\": \"How are you doing , Bob\"}\n",
"json_data = {\"vld_key\": peer_vld_key, \"message\": \"How are you doing , Bob\"}\n",
"app_message = requests.post(f\"http://{host}:{port}/app_message\", json=json_data)"
]
},
Expand All @@ -185,7 +196,7 @@
"source": [
"res = requests.get(\n",
" f\"http://{host}:{port}/proxy\",\n",
" json={\"url\": \"https://www.google.com\", \"method\": \"GET\", \"dht_key\": self_dht_key},\n",
" json={\"url\": \"https://www.google.com\", \"method\": \"GET\", \"vld_key\": self_vld_key},\n",
")"
]
},
Expand Down
33 changes: 22 additions & 11 deletions notebooks/Testing/Veilid/Bob-Python-Server.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
"metadata": {},
"outputs": [],
"source": [
"res = requests.post(f\"http://{host}:{port}/generate_dht_key\")"
"res = requests.post(f\"http://{host}:{port}/generate_vld_key\")"
]
},
{
Expand All @@ -65,7 +65,7 @@
"metadata": {},
"outputs": [],
"source": [
"res = requests.get(f\"http://{host}:{port}/retrieve_dht_key\")"
"res = requests.get(f\"http://{host}:{port}/retrieve_vld_key\")"
]
},
{
Expand All @@ -75,9 +75,9 @@
"metadata": {},
"outputs": [],
"source": [
"self_dht_key = res.json()[\"message\"]\n",
"self_vld_key = res.json()[\"message\"]\n",
"print(\"=\" * 30)\n",
"print(self_dht_key)\n",
"print(self_vld_key)\n",
"print(\"=\" * 30)"
]
},
Expand All @@ -89,15 +89,26 @@
"### Send AppMessage using DHT Key to Self"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3e810776-491d-4170-a9c5-bf7eaf2995bd",
"metadata": {},
"outputs": [],
"source": [
"# Cannot send messages to self, due to local routing feature not\n",
"# available in direct routing"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "538913ae-29be-41a5-9608-4c694ccb392b",
"metadata": {},
"outputs": [],
"source": [
"json_data = {\"dht_key\": self_dht_key, \"message\": \"Hello to me\"}\n",
"app_message = requests.post(f\"http://{host}:{port}/app_message\", json=json_data)"
"# json_data = {\"dht_key\": self_dht_key, \"message\": \"Hello to me\"}\n",
"# app_message = requests.post(f\"http://{host}:{port}/app_message\", json=json_data)"
]
},
{
Expand All @@ -115,8 +126,8 @@
"metadata": {},
"outputs": [],
"source": [
"json_data = {\"dht_key\": self_dht_key, \"message\": \"Hello to app call\"}\n",
"app_call = requests.post(f\"http://{host}:{port}/app_call\", json=json_data)"
"# json_data = {\"dht_key\": self_dht_key, \"message\": \"Hello to app call\"}\n",
"# app_call = requests.post(f\"http://{host}:{port}/app_call\", json=json_data)"
]
},
{
Expand All @@ -126,7 +137,7 @@
"metadata": {},
"outputs": [],
"source": [
"app_call.json()"
"# app_call.json()"
]
},
{
Expand All @@ -144,7 +155,7 @@
"metadata": {},
"outputs": [],
"source": [
"peer_dht_key = input(\"Enter Peer DHT Key\")"
"peer_vld_key = input(\"Enter Peer VLD Key\")"
]
},
{
Expand All @@ -154,7 +165,7 @@
"metadata": {},
"outputs": [],
"source": [
"json_data = {\"dht_key\": peer_dht_key, \"message\": \"Hello Alice\"}\n",
"json_data = {\"vld_key\": peer_vld_key, \"message\": \"Hello Alice\"}\n",
"app_message = requests.post(f\"http://{host}:{port}/app_message\", json=json_data)"
]
},
Expand Down
12 changes: 6 additions & 6 deletions notebooks/Testing/Veilid/Veilid-Gateway-Testing.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
"metadata": {},
"outputs": [],
"source": [
"domain_client.api.services.veilid.generate_dht_key()"
"domain_client.api.services.veilid.generate_vld_key()"
]
},
{
Expand All @@ -48,7 +48,7 @@
"metadata": {},
"outputs": [],
"source": [
"gateway_client.api.services.veilid.generate_dht_key()"
"gateway_client.api.services.veilid.generate_vld_key()"
]
},
{
Expand All @@ -69,7 +69,7 @@
"metadata": {},
"outputs": [],
"source": [
"gateway_route.dht_key"
"gateway_route.vld_key"
]
},
{
Expand All @@ -79,7 +79,7 @@
"metadata": {},
"outputs": [],
"source": [
"domain_route.dht_key"
"domain_route.vld_key"
]
},
{
Expand All @@ -99,7 +99,7 @@
"metadata": {},
"outputs": [],
"source": [
"domain_client.peers[0].node_routes[0].dht_key"
"domain_client.peers[0].node_routes[0].vld_key"
]
},
{
Expand All @@ -109,7 +109,7 @@
"metadata": {},
"outputs": [],
"source": [
"gateway_client.api.services.network.get_all_peers()[0].node_routes[0].dht_key"
"gateway_client.api.services.network.get_all_peers()[0].node_routes[0].vld_key"
]
},
{
Expand Down
35 changes: 18 additions & 17 deletions packages/grid/veilid/server/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@

# relative
from .models import ResponseModel
from .veilid_core import VeilidConnectionSingleton
from .veilid_connection_singleton import VeilidConnectionSingleton
from .veilid_core import app_call
from .veilid_core import app_message
from .veilid_core import generate_dht_key
from .veilid_core import generate_vld_key
from .veilid_core import healthcheck
from .veilid_core import retrieve_dht_key
from .veilid_core import retrieve_vld_key

# Logging Configuration
log_level = os.getenv("APP_LOG_LEVEL", "INFO").upper()
Expand All @@ -45,41 +45,42 @@ async def healthcheck_endpoint() -> ResponseModel:
return ResponseModel(message="FAIL")


@app.post("/generate_dht_key", response_model=ResponseModel)
async def generate_dht_key_endpoint() -> ResponseModel:
@app.post("/generate_vld_key", response_model=ResponseModel)
async def generate_vld_key_endpoint() -> ResponseModel:
try:
res = await generate_dht_key()
res = await generate_vld_key()
return ResponseModel(message=res)
except Exception as e:
raise HTTPException(status_code=500, detail=f"Failed to generate DHT key: {e}")
raise HTTPException(status_code=500, detail=f"Failed to generate VLD key: {e}")


@app.get("/retrieve_dht_key", response_model=ResponseModel)
async def retrieve_dht_key_endpoint() -> ResponseModel:
@app.get("/retrieve_vld_key", response_model=ResponseModel)
async def retrieve_vld_key_endpoint() -> ResponseModel:
try:
res = await retrieve_dht_key()
res = await retrieve_vld_key()
return ResponseModel(message=res)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))


@app.post("/app_message", response_model=ResponseModel)
async def app_message_endpoint(
request: Request, dht_key: Annotated[str, Body()], message: Annotated[bytes, Body()]
request: Request, vld_key: Annotated[str, Body()], message: Annotated[bytes, Body()]
) -> ResponseModel:
try:
res = await app_message(dht_key=dht_key, message=message)
logger.info("Received app_message request")
res = await app_message(vld_key=vld_key, message=message)
return ResponseModel(message=res)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))


@app.post("/app_call")
async def app_call_endpoint(
request: Request, dht_key: Annotated[str, Body()], message: Annotated[bytes, Body()]
request: Request, vld_key: Annotated[str, Body()], message: Annotated[bytes, Body()]
) -> Response:
try:
res = await app_call(dht_key=dht_key, message=message)
res = await app_call(vld_key=vld_key, message=message)
return Response(res, media_type="application/octet-stream")
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
Expand All @@ -92,11 +93,11 @@ async def proxy(request: Request) -> Response:
request_data = await request.json()
logger.info(f"Request URL: {request_data}")

dht_key = request_data.get("dht_key")
request_data.pop("dht_key")
vld_key = request_data.get("vld_key")
request_data.pop("vld_key")
message = json.dumps(request_data).encode()

res = await app_call(dht_key=dht_key, message=message)
res = await app_call(vld_key=vld_key, message=message)
decompressed_res = lzma.decompress(res)
return Response(decompressed_res, media_type="application/octet-stream")

Expand Down
Loading

0 comments on commit 47c132c

Please sign in to comment.