Dataset and Action Object Deletion APIs

notebooks/tutorials/model-training/01-data-scientist-submit-code.ipynb

@@ -103,7 +103,8 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "assert training_images.data is None"
+ "assert isinstance(training_images.data, sy.SyftError)\n",
+ "training_labels.data"
  ]
  },
  {

packages/syft/src/syft/client/datasite_client.py

@@ -108,7 +108,7 @@ def upload_dataset(self, dataset: CreateDataset) -> SyftSuccess | SyftError:
  asset = dataset.asset_list[i]
  dataset.asset_list[i] = add_default_uploader(user, asset)
 
- dataset._check_asset_must_contain_mock()
+ # dataset._check_asset_must_contain_mock()
  dataset_size: float = 0.0
 
  # TODO: Refactor so that object can also be passed to generate warnings

packages/syft/src/syft/service/action/action_service.py

@@ -2,6 +2,7 @@
 import importlib
 import logging
 from typing import Any
+from typing import cast
 
 # third party
 import numpy as np
@@ -289,10 +290,6 @@ def _get(
  resolve_nested: bool = True,
  ) -> Result[ActionObject, str]:
  """Get an object from the action store"""
- # stdlib
-
- # relative
-
  result = self.store.get(
  uid=uid, credentials=context.credentials, has_permission=has_permission
  )
@@ -945,12 +942,114 @@ def exists(
 
  @service_method(path="action.delete", name="delete", roles=ADMIN_ROLE_LEVEL)
  def delete(
- self, context: AuthedServiceContext, uid: UID
+ self, context: AuthedServiceContext, uid: UID, soft_delete: bool = False
  ) -> SyftSuccess | SyftError:
- res = self.store.delete(context.credentials, uid)
- if res.is_err():
- return SyftError(message=res.err())
- return SyftSuccess(message="Great Success!")
+ get_res = self.store.get(uid=uid, credentials=context.credentials)
+ if get_res.is_err():
+ return SyftError(message=get_res.err())
+ obj: ActionObject | TwinObject = get_res.ok()
+ return_msg = []
+
+ # delete any associated blob storage entry object to the action object
+ blob_del_res = self._delete_blob_storage_entry(context=context, obj=obj)
+ if isinstance(blob_del_res, SyftError):
+ return SyftError(message=blob_del_res.message)
+ return_msg.append(blob_del_res.message)
+
+ # delete the action object from the action store
+ store_del_res = self._delete_from_action_store(
+ context=context, uid=obj.id, soft_delete=soft_delete
+ )
+ if isinstance(store_del_res, SyftError):
+ return SyftError(message=store_del_res.message)
+ return_msg.append(store_del_res.message)
+
+ return SyftSuccess(message="\n".join(return_msg))
+
+ def _delete_blob_storage_entry(
+ self,
+ context: AuthedServiceContext,
+ obj: TwinObject | ActionObject,
+ ) -> SyftSuccess | SyftError:
+ deleted_blob_ids = []
+ blob_store_service = cast(
+ BlobStorageService, context.server.get_service(BlobStorageService)
+ )
+
+ if isinstance(obj, ActionObject) and obj.syft_blob_storage_entry_id:
+ blob_del_res = blob_store_service.delete(
+ context=context, uid=obj.syft_blob_storage_entry_id
+ )
+ if isinstance(blob_del_res, SyftError):
+ return SyftError(message=blob_del_res.message)
+ deleted_blob_ids.append(obj.syft_blob_storage_entry_id)
+
+ if isinstance(obj, TwinObject):
+ if obj.private.syft_blob_storage_entry_id:
+ blob_del_res = blob_store_service.delete(
+ context=context, uid=obj.private.syft_blob_storage_entry_id
+ )
+ if isinstance(blob_del_res, SyftError):
+ return SyftError(message=blob_del_res.message)
+ deleted_blob_ids.append(obj.private.syft_blob_storage_entry_id)
+
+ if obj.mock.syft_blob_storage_entry_id:
+ blob_del_res = blob_store_service.delete(
+ context=context, uid=obj.mock.syft_blob_storage_entry_id
+ )
+ if isinstance(blob_del_res, SyftError):
+ return SyftError(message=blob_del_res.message)
+ deleted_blob_ids.append(obj.mock.syft_blob_storage_entry_id)
+
+ message = f"Deleted blob storage entries: {', '.join(str(blob_id) for blob_id in deleted_blob_ids)}"
+
+ return SyftSuccess(message=message)
+
+ def _delete_from_action_store(
+ self,
+ context: AuthedServiceContext,
+ uid: UID,
+ soft_delete: bool = False,
+ ) -> SyftSuccess | SyftError:
+ if soft_delete:
+ get_res = self.store.get(uid=uid, credentials=context.credentials)
+ if get_res.is_err():
+ return SyftError(message=get_res.err())
+ obj: ActionObject | TwinObject = get_res.ok()
+
+ if isinstance(obj, TwinObject):
+ res = self._soft_delete_action_obj(
+ context=context, action_obj=obj.private
+ )
+ if res.is_err():
+ return SyftError(message=res.err())
+ res = self._soft_delete_action_obj(context=context, action_obj=obj.mock)
+ if res.is_err():
+ return SyftError(message=res.err())
+
+ if isinstance(obj, ActionObject):
+ res = self._soft_delete_action_obj(context=context, action_obj=obj)
+ if res.is_err():
+ return SyftError(message=res.err())
+ else:
+ res = self.store.delete(credentials=context.credentials, uid=uid)
+ if res.is_err():
+ return SyftError(message=res.err())
+
+ return SyftSuccess(message=f"Action object with uid '{uid}' deleted.")
+
+ def _soft_delete_action_obj(
+ self, context: AuthedServiceContext, action_obj: ActionObject
+ ) -> Result[ActionObject, str]:
+ action_obj.syft_action_data_cache = None
+ res = action_obj._save_to_blob_storage()
+ if isinstance(res, SyftError):
+ return Err(res.message)
+ set_result = self._set(
+ context=context,
+ action_object=action_obj,
+ )
+ return set_result
 
 
 def resolve_action_args(

packages/syft/src/syft/service/blob_storage/service.py

@@ -352,30 +352,36 @@ def mark_write_complete(
  def delete(
  self, context: AuthedServiceContext, uid: UID
  ) -> SyftSuccess | SyftError:
- result = self.stash.get_by_uid(context.credentials, uid=uid)
- if result.is_ok():
- obj = result.ok()
+ get_res = self.stash.get_by_uid(context.credentials, uid=uid)
+ if get_res.is_err():
+ return SyftError(message=get_res.err())
 
- if obj is None:
- return SyftError(
- message=f"No blob storage entry exists for uid: {uid}, or you have no permissions to read it"
- )
-
- try:
- with context.server.blob_storage_client.connect() as conn:
- file_unlinked_result = conn.delete(obj.location)
- except Exception as e:
- return SyftError(message=f"Failed to delete file: {e}")
+ obj = get_res.ok()
+ if obj is None:
+ return SyftError(
+ message=f"No blob storage entry exists for uid: {uid}, "
+ f"or you have no permissions to read it"
+ )
 
- if isinstance(file_unlinked_result, SyftError):
- return file_unlinked_result
- blob_storage_entry_deleted = self.stash.delete(
- context.credentials, UIDPartitionKey.with_obj(uid), has_permission=True
+ try:
+ with context.server.blob_storage_client.connect() as conn:
+ file_unlinked_result = conn.delete(obj.location)
+ if isinstance(file_unlinked_result, SyftError):
+ return file_unlinked_result
+ except Exception as e:
+ return SyftError(
+ message=f"Failed to delete blob file with id '{uid}'. Error: {e}"
  )
- if blob_storage_entry_deleted.is_ok():
- return file_unlinked_result
 
- return SyftError(message=result.err())
+ blob_entry_delete_res = self.stash.delete(
+ context.credentials, UIDPartitionKey.with_obj(uid), has_permission=True
+ )
+ if blob_entry_delete_res.is_err():
+ return SyftError(message=blob_entry_delete_res.err())
+
+ return SyftSuccess(
+ message=f"Blob storage entry with id '{uid}' deleted successfully."
+ )
 
 
 TYPE_TO_SERVICE[BlobStorageEntry] = BlobStorageEntry

packages/syft/src/syft/service/dataset/dataset.py

@@ -20,14 +20,17 @@
 from typing_extensions import Self
 
 # relative
+from ...client.api import APIRegistry
 from ...serde.serializable import serializable
 from ...store.document_store import PartitionKey
 from ...types.datetime import DateTime
 from ...types.dicttuple import DictTuple
+from ...types.syft_object import PartialSyftObject
 from ...types.syft_object import SYFT_OBJECT_VERSION_1
 from ...types.syft_object import SyftObject
 from ...types.transforms import TransformContext
 from ...types.transforms import generate_id
+from ...types.transforms import make_set_default
 from ...types.transforms import transform
 from ...types.transforms import validate_url
 from ...types.uid import UID
@@ -239,9 +242,6 @@ def __eq__(self, other: object) -> bool:
 
  @property
  def pointer(self) -> Any:
- # relative
- from ...client.api import APIRegistry
-
  api = APIRegistry.api_for(
  server_uid=self.server_uid,
  user_verify_key=self.syft_client_verify_key,
@@ -251,16 +251,15 @@ def pointer(self) -> Any:
 
  @property
  def mock(self) -> SyftError | Any:
- # relative
- from ...client.api import APIRegistry
-
  api = APIRegistry.api_for(
  server_uid=self.server_uid,
  user_verify_key=self.syft_client_verify_key,
  )
  if api is None:
  return SyftError(message=f"You must login to {self.server_uid}")
  result = api.services.action.get_mock(self.action_id)
+ if isinstance(result, SyftError):
+ return result
  try:
  if isinstance(result, SyftObject):
  return result.syft_action_data
@@ -282,7 +281,6 @@ def has_permission(self, data_result: Any) -> bool:
  @property
  def data(self) -> Any:
  # relative
- from ...client.api import APIRegistry
 
  api = APIRegistry.api_for(
  server_uid=self.server_uid,
@@ -291,6 +289,8 @@ def data(self) -> Any:
  if api is None or api.services is None:
  return None
  res = api.services.action.get(self.action_id)
+ if isinstance(res, str):
+ return SyftError(message=f"Could not access private data. {str(res)}")
  if self.has_permission(res):
  return res.syft_action_data
  else:
@@ -471,6 +471,7 @@ class Dataset(SyftObject):
  created_at: DateTime = DateTime.now()
  uploader: Contributor
  summary: str | None = None
+ to_be_deleted: bool = False
 
  __attr_searchable__ = [
  "name",
@@ -523,6 +524,8 @@ def _repr_html_(self) -> Any:
  """
  else:
  description_info_message = ""
+ if self.to_be_deleted:
+ return "This dataset has been marked for deletion. The underlying data may be not available."
  return f"""
  <style>
  {FONT_CSS}
@@ -549,8 +552,7 @@ def _repr_html_(self) -> Any:
  """
 
  def action_ids(self) -> list[UID]:
- data = [asset.action_id for asset in self.asset_list if asset.action_id]
- return data
+ return [asset.action_id for asset in self.asset_list if asset.action_id]
 
  @property
  def assets(self) -> DictTuple[str, Asset]:
@@ -639,9 +641,6 @@ class CreateDataset(Dataset):
 
  model_config = ConfigDict(validate_assignment=True, extra="forbid")
 
- def _check_asset_must_contain_mock(self) -> None:
- _check_asset_must_contain_mock(self.asset_list)
-
  @field_validator("asset_list")
  @classmethod
  def __assets_must_contain_mock(
@@ -650,6 +649,13 @@ def __assets_must_contain_mock(
  _check_asset_must_contain_mock(asset_list)
  return asset_list
 
+ @field_validator("to_be_deleted")
+ @classmethod
+ def __to_be_deleted_must_be_false(cls, v: bool) -> bool:
+ if v is True:
+ raise ValueError("to_be_deleted must be False")
+ return v
+
  def set_description(self, description: str) -> None:
  self.description = MarkdownDescription(text=description)
 
@@ -876,8 +882,13 @@ def createdataset_to_dataset() -> list[Callable]:
  validate_url,
  convert_asset,
  add_current_date,
+ make_set_default("to_be_deleted", False), # explicitly set it to False
  ]
 
 
-class DatasetUpdate:
- pass
+class DatasetUpdate(PartialSyftObject):
+ __canonical_name__ = "DatasetUpdate"
+ __version__ = SYFT_OBJECT_VERSION_1
+
+ name: str
+ to_be_deleted: bool