[Ready for Review] fix bug where client can not access foreach stack (#…

…1766)

* fix bug where client can not access foreach stack

metaflow/__init__.py

@@ -143,6 +143,9 @@ class and related decorators.
  DataArtifact,
 )
 
+# Import data class within tuple_util but not introduce new symbols.
+from . import tuple_util
+
 __version_addl__ = []
 _ext_debug("Loading top-level modules")
 for m in _tl_modules:

metaflow/plugins/datatools/s3/s3.py

@@ -21,14 +21,14 @@
  TEMPDIR,
 )
 from metaflow.util import (
- namedtuple_with_defaults,
  is_stringish,
  to_bytes,
  to_unicode,
  to_fileobj,
  url_quote,
  url_unquote,
 )
+from metaflow.tuple_util import namedtuple_with_defaults
 from metaflow.exception import MetaflowException
 from metaflow.debug import debug
 import metaflow.tracing as tracing

metaflow/task.py

@@ -23,18 +23,7 @@
 from .clone_util import clone_task_helper
 from .metaflow_current import current
 from metaflow.tracing import get_trace_id
-from metaflow.util import namedtuple_with_defaults
-
-foreach_frame_field_list = [
- ("step", str),
- ("var", str),
- ("num_splits", int),
- ("index", int),
- ("value", str),
-]
-ForeachFrame = namedtuple_with_defaults(
- "ForeachFrame", foreach_frame_field_list, (None,) * (len(foreach_frame_field_list))
-)
+from metaflow.tuple_util import ForeachFrame
 
 # Maximum number of characters of the foreach path that we store in the metadata.
 MAX_FOREACH_PATH_LENGTH = 256

metaflow/tuple_util.py

@@ -0,0 +1,27 @@
+# Keep this file minimum dependency as this will be imported by metaflow at bootup.
+def namedtuple_with_defaults(typename, field_descr, defaults=()):
+ from typing import NamedTuple
+
+ T = NamedTuple(typename, field_descr)
+ T.__new__.__defaults__ = tuple(defaults)
+
+ # Adding the following to ensure the named tuple can be (un)pickled correctly.
+ import __main__
+
+ setattr(__main__, T.__name__, T)
+ T.__module__ = "__main__"
+ return T
+
+
+# Define the namedtuple with default here if they need to be accessible in client
+# (and w/o a real flow).
+foreach_frame_field_list = [
+ ("step", str),
+ ("var", str),
+ ("num_splits", int),
+ ("index", int),
+ ("value", str),
+]
+ForeachFrame = namedtuple_with_defaults(
+ "ForeachFrame", foreach_frame_field_list, (None,) * (len(foreach_frame_field_list))
+)

metaflow/util.py

@@ -51,21 +51,6 @@ def unquote_bytes(x):
  from shlex import quote as _quote
 
 
-from typing import NamedTuple
-
-
-def namedtuple_with_defaults(typename, field_descr, defaults=()):
- T = NamedTuple(typename, field_descr)
- T.__new__.__defaults__ = tuple(defaults)
-
- # Adding the following to ensure the named tuple can be (un)pickled correctly.
- import __main__
-
- setattr(__main__, T.__name__, T)
- T.__module__ = "__main__"
- return T
-
-
 class TempDir(object):
  # Provide a temporary directory since Python 2.7 does not have it inbuilt
  def __enter__(self):