refactoring bulky requirements into pipeline extra

Dockerfile

@@ -71,13 +71,16 @@ COPY . eta/
 
 ARG TENSORFLOW_VERSION
 RUN pip --no-cache-dir  install --upgrade pip setuptools \
-    && pip --no-cache-dir install -r eta/requirements.txt \
+    && pip --no-cache-dir install -r eta/requirements/common.txt \
+    && pip --no-cache-dir install -r eta/requirements/pipeline.txt \
+    && pip --no-cache-dir install -r eta/requirements/storage.txt \
     && pip --no-cache-dir install --upgrade setuptools \
     && pip --no-cache-dir install -e eta/. \
     && pip --no-cache-dir install -I $TENSORFLOW_VERSION \
     && pip --no-cache-dir install --upgrade numpy==1.16.0 \
     && pip --no-cache-dir install -e eta/eta/tensorflow/darkflow/. \
     && pip --no-cache-dir install pycocotools \
+    && pip --no-cache-dir install protobuf \
     && curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip \
     && unzip protoc-3.6.1-linux-x86_64.zip -d protoc3 \
     && rm -rf protoc-3.6.1-linux-x86_64.zip \

eta/core/cli.py

@@ -35,14 +35,14 @@
 import eta.core.builder as etab
 import eta.constants as etac
 import eta.core.logging as etal
-import eta.core.metadata as etame
 import eta.core.models as etamode
 import eta.core.module as etamodu
 import eta.core.pipeline as etap
 import eta.core.serial as etase
 import eta.core.utils as etau
 import eta.core.web as etaw
 
+etame = etau.lazy_import("eta.core.metadata")
 etast = etau.lazy_import("eta.core.storage")
 etat = etau.lazy_import("eta.core.tfutils")
 

eta/core/diagram.py

@@ -75,6 +75,11 @@ def render(self, path, keep_diag_file=False):
             logger.info("Generating block diagram '%s'", svg_path)
             args = ["blockdiag", "-Tsvg", "-o", svg_path, blockdiag_path]
             etau.communicate_or_die(args)
+        except etau.ExecutableNotFoundError:
+            raise etau.PackageError(
+                "You must run pip install voxel51-eta[pipeline] in order to "
+                "use this feature"
+            )
         finally:
             if not keep_diag_file:
                 etau.delete_file(blockdiag_path)

eta/core/metadata.py

@@ -25,13 +25,20 @@
 import re
 import sys
 
-from docutils.core import publish_doctree
-from docutils.nodes import paragraph, field_list, field_name, field_body
-
-from sphinx.ext.napoleon.docstring import GoogleDocstring
-from sphinx.util.docstrings import prepare_docstring
-
 import eta.core.module as etam
+import eta.core.utils as etau
+
+try:
+    from docutils.core import publish_doctree
+    from docutils.nodes import paragraph, field_list, field_name, field_body
+
+    from sphinx.ext.napoleon.docstring import GoogleDocstring
+    from sphinx.util.docstrings import prepare_docstring
+except ImportError:
+    raise etau.PackageError(
+        "You must run pip install voxel51-eta[pipeline] in order to "
+        "use this feature"
+    )
 
 
 logger = logging.getLogger(__name__)

eta/tensorflow/install_models.bash

@@ -32,6 +32,9 @@ fi
 
 cd "${MODELS_DIR}"
 
+echo "Installing protobuf"
+pip install protobuf
+
 if command -v protoc &> /dev/null; then
     echo "Found protoc"
 else

install.bash

@@ -190,12 +190,35 @@ fi
 
 
 MSG "Installing Python packages"
+CRITICAL pip install -r requirements.txt
+
+
+MSG "Installing ETA"
+if [ ${DEV_INSTALL} = true ]; then
+    CRITICAL pip install -e .
+else
+    CRITICAL pip install .
+fi
+
+
+# @note(lite) handle lite installation
+if [[ ${LITE_INSTALL} = true ]]; then
+    EXIT "LITE INSTALLATION COMPLETE"
+fi
+
+
+MSG "Installing storage extras"
+CRITICAL pip install -r requirements/storage.txt
+
+
+MSG "Installing pipeline extras"
+CRITICAL pip install -r requirements/pipeline.txt
+
+
 if [ ${DEV_INSTALL} = true ]; then
     MSG "Performing dev install"
     CRITICAL pip install -r requirements/dev.txt
     CRITICAL pre-commit install
-else
-    CRITICAL pip install -r requirements.txt
 fi
 
 
@@ -229,14 +252,6 @@ else
 fi
 
 
-MSG "Installing ETA"
-if [ ${DEV_INSTALL} = true ]; then
-    CRITICAL pip install -e .
-else
-    CRITICAL pip install .
-fi
-
-
 # Install ffmpeg
 INFO command -v ffmpeg
 if [ $? -eq 0 ]; then
@@ -273,12 +288,6 @@ else
 fi
 
 
-# @note(lite) handle lite installation
-if [[ ${LITE_INSTALL} = true ]]; then
-    EXIT "LITE INSTALLATION COMPLETE"
-fi
-
-
 MSG "Initializing submodules"
 git submodule init
 git submodule update

requirements/common.txt

@@ -1,26 +1,17 @@
 argcomplete==1.11.0
-blockdiag==1.5.3
-boto3==1.10.9
 contextlib2==0.5.5
 Cython==0.28.5
 dill==0.2.7.1
 future==0.16.0
 glob2==0.6
-google-api-python-client==1.6.5
-google-auth-httplib2==0.0.3
-google-cloud-storage==1.7.0
-httplib2==0.15.0
 importlib-metadata==1.3.0; python_version<"3.8"
-Jinja2==2.11.3
 lxml==4.6.3
 ndjson==0.3.1
 numpy==1.16.3
 opencv-python-headless==4.1.0.25
 packaging==19.2
 patool==1.12
 Pillow==8.1.1
-protobuf==3.6.1
-pysftp==0.2.9
 python-dateutil==2.7.0
 pytz==2019.3
 requests-toolbelt==0.8.0
@@ -33,6 +24,5 @@ setuptools==45.2.0;python_version>="3"
 simplejson==3.8.1
 six==1.11.0
 sortedcontainers==2.1.0
-Sphinx==2.4.4
 tabulate==0.8.5
 tzlocal==2.0.0

requirements/dev.txt

@@ -1,10 +1,9 @@
--r common.txt
-
 ipython==5.8.0;python_version<"3"
 ipython==7.8.0;python_version>="3"
 m2r==0.2.1
 pre-commit==2.0.1
 pylint==1.9.4;python_version<"3"
 pylint==2.3.1;python_version>="3"
+Sphinx==2.4.4
 sphinxcontrib-napoleon==0.7
 sphinx-rtd-theme==0.4.3

setup.py

@@ -68,11 +68,11 @@ def get_version():
         "six",
         "scikit-image",
         "sortedcontainers",
-        "Sphinx",
         "tabulate",
         "tzlocal",
     ],
     extras_require={
+        "pipeline": ["blockdiag", "Sphinx", "sphinxcontrib-napoleon"],
         "storage": [
             "boto3",
             "google-api-python-client",