Merge pull request #8936 from OpenMined/function-factories-demo

Function factories demo

notebooks/api/0.8/02-review-code-and-approve.ipynb

@@ -290,6 +290,21 @@
  "print(op.policy_code)"
  ]
  },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Policies provided by Syft are available before approving the code,\n",
+ "# Custom policies are only safe to use once the code is approved.\n",
+ "\n",
+ "assert func.output_policy is not None\n",
+ "assert func.input_policy is not None\n",
+ "\n",
+ "func.output_policy"
+ ]
+ },
  {
  "cell_type": "markdown",
  "metadata": {},

notebooks/api/0.8/05-custom-policy.ipynb

@@ -508,6 +508,18 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "23",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Custom policies need to be approved before they can be viewed and used\n",
+ "assert func.input_policy is None\n",
+ "assert func.output_policy is None"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "24",
  "metadata": {
  "tags": []
  },
@@ -520,7 +532,7 @@
  {
  "cell_type": "code",
  "execution_count": null,
- "id": "24",
+ "id": "25",
  "metadata": {},
  "outputs": [],
  "source": [
@@ -530,7 +542,18 @@
  {
  "cell_type": "code",
  "execution_count": null,
- "id": "25",
+ "id": "26",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "assert func.input_policy is not None\n",
+ "assert func.output_policy is not None"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "27",
  "metadata": {
  "tags": []
  },
@@ -543,7 +566,7 @@
  {
  "cell_type": "code",
  "execution_count": null,
- "id": "26",
+ "id": "28",
  "metadata": {},
  "outputs": [],
  "source": [
@@ -554,7 +577,7 @@
  {
  "cell_type": "code",
  "execution_count": null,
- "id": "27",
+ "id": "29",
  "metadata": {
  "tags": []
  },
@@ -566,7 +589,7 @@
  {
  "cell_type": "code",
  "execution_count": null,
- "id": "28",
+ "id": "30",
  "metadata": {
  "tags": []
  },
@@ -578,7 +601,7 @@
  {
  "cell_type": "code",
  "execution_count": null,
- "id": "29",
+ "id": "31",
  "metadata": {
  "tags": []
  },
@@ -594,7 +617,7 @@
  {
  "cell_type": "code",
  "execution_count": null,
- "id": "30",
+ "id": "32",
  "metadata": {
  "tags": []
  },

notebooks/scenarios/bigquery/.gitignore

@@ -0,0 +1 @@
+service_account.json

packages/grid/backend/backend.dockerfile

@@ -1,6 +1,6 @@
 ARG PYTHON_VERSION="3.12"
 ARG UV_VERSION="0.2.13-r0"
-ARG TORCH_VERSION="2.3.0"
+ARG TORCH_VERSION="2.2.2"
 
 # wolfi-os pkg definition links
 # https://github.com/wolfi-dev/os/blob/main/python-3.12.yaml

packages/syft/setup.cfg

@@ -87,7 +87,7 @@ data_science =
  evaluate==0.4.2
  recordlinkage==0.16
  # backend.dockerfile installs torch separately, so update the version over there as well!
- torch==2.3.0
+ torch==2.2.2
 
 dev =
  %(test_plugins)s

packages/syft/src/syft/__init__.py

@@ -59,9 +59,11 @@
 from .service.dataset.dataset import CreateAsset as Asset # noqa: F401
 from .service.dataset.dataset import CreateDataset as Dataset # noqa: F401
 from .service.notification.notifications import NotificationStatus # noqa: F401
+from .service.policy.policy import CreatePolicyRuleConstant as Constant # noqa: F401
 from .service.policy.policy import CustomInputPolicy # noqa: F401
 from .service.policy.policy import CustomOutputPolicy # noqa: F401
 from .service.policy.policy import ExactMatch # noqa: F401
+from .service.policy.policy import MixedInputPolicy # noqa: F401
 from .service.policy.policy import SingleExecutionExactOutput # noqa: F401
 from .service.policy.policy import UserInputPolicy # noqa: F401
 from .service.policy.policy import UserOutputPolicy # noqa: F401

packages/syft/src/syft/client/client.py

@@ -757,6 +757,22 @@ def login_as_guest(self) -> Self:
 
  return _guest_client
 
+ def login_as(self, email: str) -> Self:
+ user_private_key = self.api.services.user.key_for_email(email=email)
+ if not isinstance(user_private_key, UserPrivateKey):
+ return user_private_key
+ if self.metadata is not None:
+ print(
+ f"Logged into <{self.name}: {self.metadata.node_side_type.capitalize()}-side "
+ f"{self.metadata.node_type.capitalize()}> as {email}"
+ )
+
+ return self.__class__(
+ connection=self.connection,
+ credentials=user_private_key.signing_key,
+ metadata=self.metadata,
+ )
+
  def login(
  self,
  email: str | None = None,

packages/syft/src/syft/client/domain_client.py

@@ -178,7 +178,6 @@ def get_sync_state(self) -> SyncState | SyftError:
  for uid, obj in state.objects.items():
  if isinstance(obj, ActionObject):
  obj = obj.refresh_object(resolve_nested=False)
- obj.reload_cache()
  state.objects[uid] = obj
  return state
 
@@ -190,8 +189,10 @@ def apply_state(self, resolved_state: ResolvedSyncState) -> SyftSuccess | SyftEr
  action_objects = [x for x in items if isinstance(x, ActionObject)]
 
  for action_object in action_objects:
+ action_object.reload_cache()
  # NOTE permissions are added separately server side
  action_object._send(self.id, self.verify_key, add_storage_permission=False)
+ action_object._clear_cache()
 
  ignored_batches = resolved_state.ignored_batches
 

packages/syft/src/syft/node/node.py

@@ -1446,8 +1446,10 @@ def add_queueitem_to_queue(
  )
 
  # 🟡 TODO 36: Needs distributed lock
+ job_res = self.job_stash.set(credentials, job)
+ if job_res.is_err():
+ return SyftError(message=f"{job_res.err()}")
  self.queue_stash.set_placeholder(credentials, queue_item)
- self.job_stash.set(credentials, job)
 
  log_service = self.get_service("logservice")
 
@@ -1465,12 +1467,17 @@ def _get_existing_user_code_jobs(
  )
 
  def _is_usercode_call_on_owned_kwargs(
- self, context: AuthedServiceContext, api_call: SyftAPICall
+ self,
+ context: AuthedServiceContext,
+ api_call: SyftAPICall,
+ user_code_id: UID,
  ) -> bool:
  if api_call.path != "code.call":
  return False
  user_code_service = self.get_service("usercodeservice")
- return user_code_service.is_execution_on_owned_args(api_call.kwargs, context)
+ return user_code_service.is_execution_on_owned_args(
+ context, user_code_id, api_call.kwargs
+ )
 
  def add_api_call_to_queue(
  self, api_call: SyftAPICall, parent_job_id: UID | None = None
@@ -1493,18 +1500,17 @@ def add_api_call_to_queue(
  action = None
  if is_user_code:
  action = Action.from_api_call(unsigned_call)
+ user_code_id = action.user_code_id
 
  is_usercode_call_on_owned_kwargs = self._is_usercode_call_on_owned_kwargs(
- context, unsigned_call
+ context, unsigned_call, user_code_id
  )
  # Low side does not execute jobs, unless this is a mock execution
  if (
  not is_usercode_call_on_owned_kwargs
  and self.node_side_type == NodeSideType.LOW_SIDE
  ):
- existing_jobs = self._get_existing_user_code_jobs(
- context, action.user_code_id
- )
+ existing_jobs = self._get_existing_user_code_jobs(context, user_code_id)
  if isinstance(existing_jobs, SyftError):
  return existing_jobs
  elif len(existing_jobs) > 0:

packages/syft/src/syft/protocol/protocol_version.json

@@ -298,6 +298,62 @@
  "hash": "9b8ab2d513d84006bdd1329cd0bb636e7e62100a6227d8b772a5bf7c0c45b72f",
  "action": "add"
  }
+ },
+ "PolicyRule": {
+ "1": {
+ "version": 1,
+ "hash": "31a982b94654ce27ad27a6622c6fa26dfe3f759a7824ac21d104390f10a5aa82",
+ "action": "add"
+ }
+ },
+ "CreatePolicyRule": {
+ "1": {
+ "version": 1,
+ "hash": "9b82e36c63e10c5b7b76b3b8ec1da1d2dfdce39f2cce98603a418ec221621874",
+ "action": "add"
+ }
+ },
+ "CreatePolicyRuleConstant": {
+ "1": {
+ "version": 1,
+ "hash": "9e821ddd383b6472f95dad2b56ebaefad225ff49c96b89b4ce35dc99c422ba76",
+ "action": "add"
+ }
+ },
+ "Matches": {
+ "1": {
+ "version": 1,
+ "hash": "d1e875a6332a481458e83db364dfdf92bd34a87093d9762dfe8e136e5088bc4e",
+ "action": "add"
+ }
+ },
+ "PreFill": {
+ "1": {
+ "version": 1,
+ "hash": "22c38b8ad68409493810362e6c48822d3e2919760f26eba2d1de3f2ad394e314",
+ "action": "add"
+ }
+ },
+ "UserOwned": {
+ "1": {
+ "version": 1,
+ "hash": "b5cbb44d742fa51b9adf2a48bb56d9ff5ca82a25f8568a2505961bd906d9d084",
+ "action": "add"
+ }
+ },
+ "MixedInputPolicy": {
+ "1": {
+ "version": 1,
+ "hash": "0e84e4c91e378717e1a4703574b07e3b1e6a3e5707401b4e0cc8d30088a506b9",
+ "action": "add"
+ }
+ },
+ "Request": {
+ "3": {
+ "version": 3,
+ "hash": "ba9ebb04cc3e8b3ae3302fd42a67e47261a0a330bae5f189d8f4819cf2804711",
+ "action": "add"
+ }
  }
  }
  }

packages/syft/src/syft/service/action/action_object.py

@@ -1980,7 +1980,7 @@ def _repr_markdown_(self, wrap_as_python: bool = True, indent: int = 0) -> str:
  else self.syft_action_data_cache.__repr__()
  )
 
- return f"```python\n{res}\n{data_repr_}\n```\n"
+ return f"\n**{res}**\n\n{data_repr_}\n"
 
  def _data_repr(self) -> str | None:
  if isinstance(self.syft_action_data_cache, ActionDataEmpty):