Send small variables to storage without blob storage

packages/syft/src/syft/client/domain_client.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 import re
 from string import Template
+import sys
 from typing import TYPE_CHECKING
 from typing import cast
 
@@ -17,6 +18,7 @@
 # relative
 from ..abstract_node import NodeSideType
 from ..serde.serializable import serializable
+from ..serde.serialize import _serialize as serialize
 from ..service.action.action_object import ActionObject
 from ..service.code_history.code_history import CodeHistoriesDict
 from ..service.code_history.code_history import UsersCodeHistoriesDict
@@ -134,6 +136,11 @@ def upload_dataset(self, dataset: CreateDataset) -> SyftSuccess | SyftError:
  syft_node_location=self.id,
  syft_client_verify_key=self.verify_key,
  )
+ serialized: bytes = serialize(twin, to_bytes=True)
+ size_mb: float = sys.getsizeof(serialized) / 1024 / 1024
+ if size_mb < 16:
+ print(f"object's size = {size_mb} (MB), less than 16 MB")
+ # TODO: if less than 16 MB, save without using blob storage
  twin._save_to_blob_storage()
  except Exception as e:
  tqdm.write(f"Failed to create twin for {asset.name}. {e}")

packages/syft/src/syft/service/action/action_object.py

@@ -498,44 +498,25 @@ def convert_to_pointers(
  # relative
  from ..dataset.dataset import Asset
 
- arg_list = []
- kwarg_dict = {}
- if args is not None:
- for arg in args:
- if (
- not isinstance(arg, ActionObject | Asset | UID)
- and api.signing_key is not None # type: ignore[unreachable]
- ):
- arg = ActionObject.from_obj( # type: ignore[unreachable]
- syft_action_data=arg,
- syft_client_verify_key=api.signing_key.verify_key,
- syft_node_location=api.node_uid,
- )
- arg.syft_node_uid = node_uid
- r = arg._save_to_blob_storage()
- if isinstance(r, SyftError):
- print(r.message)
- arg = api.services.action.set(arg)
- arg_list.append(arg)
-
- if kwargs is not None:
- for k, arg in kwargs.items():
- if (
- not isinstance(arg, ActionObject | Asset | UID)
- and api.signing_key is not None # type: ignore[unreachable]
- ):
- arg = ActionObject.from_obj( # type: ignore[unreachable]
- syft_action_data=arg,
- syft_client_verify_key=api.signing_key.verify_key,
- syft_node_location=api.node_uid,
- )
- arg.syft_node_uid = node_uid
- r = arg._save_to_blob_storage()
- if isinstance(r, SyftError):
- print(r.message)
- arg = api.services.action.set(arg)
-
- kwarg_dict[k] = arg
+ def process_arg(arg: ActionObject | Asset | UID | Any) -> Any:
+ if (
+ not isinstance(arg, ActionObject | Asset | UID)
+ and api.signing_key is not None # type: ignore[unreachable]
+ ):
+ arg = ActionObject.from_obj( # type: ignore[unreachable]
+ syft_action_data=arg,
+ syft_client_verify_key=api.signing_key.verify_key,
+ syft_node_location=api.node_uid,
+ )
+ arg.syft_node_uid = node_uid
+ r = arg._save_to_blob_storage()
+ if isinstance(r, SyftError):
+ print(r.message)
+ arg = api.services.action.set(arg)
+ return arg
+
+ arg_list = [process_arg(arg) for arg in args] if args else []
+ kwarg_dict = {k: process_arg(v) for k, v in kwargs.items()} if kwargs else {}
 
  return arg_list, kwarg_dict