Add deon and nbautoexport options

.github/workflows/tests.yml

@@ -0,0 +1,40 @@
+
+name: tests
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+  schedule:
+    # Run every Sunday
+    - cron: "0 0 * * 0"
+
+jobs:
+  build:
+    name: ${{ matrix.os }}, Python ${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: [3.7, 3.8, 3.9, "3.10"]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev-requirements.txt 
+
+      - name: Lint
+        run: |
+          make lint
+
+      - name: Run tests
+        run: |
+          pytest -vvv

.gitignore

@@ -6,4 +6,11 @@ docs/site/
 # test cache
 .cache/*
 tests/__pycache__/*
-*.pytest_cache/
+*.pytest_cache/
+*.pyc
+manual_test/
+
+# other local dev info
+.vscode/
+cookiecutter_data_science.egg-info/
+

LICENSE

@@ -5,4 +5,4 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of
 
 The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Makefile

@@ -0,0 +1,60 @@
+## GLOBALS
+
+PROJECT_NAME = cookiecutter-data-science
+PYTHON_VERSION = 3.10
+PYTHON_INTERPRETER = python
+
+
+###     UTILITIES
+_prep:
+	rm -f **/*/.DS_store
+
+
+###     DEV COMMANDS
+
+## Set up python interpreter environment
+create_environment:
+	conda create --name $(PROJECT_NAME) python=$(PYTHON_VERSION) -y
+	@echo ">>> conda env created. Activate with:\nconda activate $(PROJECT_NAME)"
+
+## Install Python Dependencies
+requirements:
+	$(PYTHON_INTERPRETER) -m pip install -r dev-requirements.txt
+
+## Format the code using isort and black
+format:
+	isort ccds hooks tests
+	black ccds hooks tests setup.py
+
+## Lint using flake8 + black
+lint:
+	flake8 ccds hooks tests setup.py
+	black --check ccds hooks tests setup.py
+
+
+###     DOCS
+
+docs-serve:
+	cd docs && mkdocs serve
+
+###     TESTS
+
+test: _prep
+	pytest -vvv
+
+test-fastest: _prep
+	pytest -vvv -FFF
+
+test-debug-last:
+	pytest --lf --pdb
+
+_clean_manual_test:
+	rm -rf manual_test
+
+manual-test: _prep _clean_manual_test
+	mkdir -p manual_test
+	cd manual_test && python -m ccds ..
+
+manual-test-debug: _prep _clean_manual_test
+	mkdir -p manual_test
+	cd manual_test && python -m pdb ../ccds/__main__.py ..

README.md

@@ -8,7 +8,7 @@ _A logical, reasonably standardized, but flexible project structure for doing an
 
 ### Requirements to use the cookiecutter template:
 -----------
- - Python 2.7 or 3.5+
+ - Python 3.7+
  - [Cookiecutter Python package](http://cookiecutter.readthedocs.org/en/latest/installation.html) >= 1.4.0: This can be installed with pip by or conda depending on how you manage your Python packages:
 
 ``` bash
@@ -44,7 +44,7 @@ which is available now.
 ### The resulting directory structure
 ------------
 
-The directory structure of your new project looks like this: 
+The directory structure of your new project looks like this:
 
 ```
 ├── LICENSE
@@ -64,6 +64,8 @@ The directory structure of your new project looks like this:
 │                         the creator's initials, and a short `-` delimited description, e.g.
 │                         `1.0-jqp-initial-data-exploration`.
 │
+├── pyproject.toml     <- Project configuration file with settings for running black; see setuptools.readthedocs.io
+│
 ├── references         <- Data dictionaries, manuals, and all other explanatory materials.
 │
 ├── reports            <- Generated analysis as HTML, PDF, LaTeX, etc.
@@ -72,9 +74,13 @@ The directory structure of your new project looks like this:
 ├── requirements.txt   <- The requirements file for reproducing the analysis environment, e.g.
 │                         generated with `pip freeze > requirements.txt`
 │
+├── setup.cfg          <- Configuration file for flake8 and pep8
+│
 ├── setup.py           <- makes project pip installable (pip install -e .) so src can be imported
-├── src                <- Source code for use in this project.
-│   ├── __init__.py    <- Makes src a Python module
+│
+├── {{ cookiecutter.module_name }}                <- Source code for use in this project.
+│   │
+│   ├── __init__.py    <- Makes {{ cookiecutter.module_name }} a Python module
 │   │
 │   ├── data           <- Scripts to download or generate data
 │   │   └── make_dataset.py
@@ -89,19 +95,24 @@ The directory structure of your new project looks like this:
 │   │
 │   └── visualization  <- Scripts to create exploratory and results oriented visualizations
 │       └── visualize.py
-│
-└── tox.ini            <- tox file with settings for running tox; see tox.readthedocs.io
+
 ```
 
 ## Contributing
 
 We welcome contributions! [See the docs for guidelines](https://drivendata.github.io/cookiecutter-data-science/#contributing).
 
-### Installing development requirements
+### Installing requirements
 ------------
 
     pip install -r requirements.txt
 
+
+### Installing development requirements
+------------
+
+    pip install -r dev-requirements.txt
+
 ### Running the tests
 ------------
 

ccds.json

@@ -0,0 +1,57 @@
+{
+    "project_name": "project_name",
+    "repo_name": "{{ cookiecutter.project_name.lower().replace(' ', '_') }}",
+    "module_name": "{{ cookiecutter.project_name.lower().replace(' ', '_').replace('-', '_') }}",
+    "author_name": "Your name (or your organization/company/team)",
+    "description": "A short description of the project.",
+    "python_version_number": "3.10",
+    "dataset_storage": [
+        {
+            "none": "none"
+        },
+        {
+            "azure": {
+                "container": "container-name"
+            }
+        },
+        {
+            "s3": {
+                "bucket": "bucket-name",
+                "aws_profile": "default"
+            }
+        },
+        {
+            "gcs": {
+                "bucket": "bucket-name"
+            }
+        }
+    ],
+    "environment_manager": [
+        "virtualenv",
+        "conda",
+        "pipenv",
+        "none"
+    ],
+    "dependency_file": [
+        "requirements.txt",
+        "environment.yml",
+        "Pipfile"
+    ],
+    "pydata_packages": [
+        "none",
+        "basic"
+    ],
+    "ethics_checklist": [
+        "yes",
+        "no"
+    ],
+    "nbautoexport": [
+        "yes",
+        "no"
+    ],
+    "open_source_license": [
+        "MIT",
+        "BSD-3-Clause",
+        "No license file"
+    ]
+}

ccds/__main__.py

@@ -0,0 +1,30 @@
+# Monkey-patch jinja to allow variables to not exist, which happens with sub-options
+import jinja2
+
+jinja2.StrictUndefined = jinja2.Undefined
+
+
+# Monkey-patch cookiecutter to allow sub-items
+from cookiecutter import prompt
+
+from ccds.monkey_patch import prompt_for_config
+
+prompt.prompt_for_config = prompt_for_config
+
+
+# monkey-patch context to point to ccds.json
+from cookiecutter import generate
+
+from ccds.monkey_patch import generate_context_wrapper
+
+generate.generate_context = generate_context_wrapper
+
+# for use in tests need monkey-patched api main
+from cookiecutter import cli
+from cookiecutter import main as api_main  # noqa: F401 referenced by tests
+
+main = cli.main
+
+
+if __name__ == "__main__":
+    main()

ccds/hook_utils/custom_config.py

@@ -0,0 +1,51 @@
+from distutils.dir_util import copy_tree
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from urllib.request import urlretrieve
+from zipfile import ZipFile
+
+from cookiecutter.vcs import clone
+
+
+def write_custom_config(user_input_config):
+    if not user_input_config:
+        return
+
+    tmp = TemporaryDirectory()
+    tmp_zip = None
+
+    print(user_input_config)
+
+    # if not absolute, test if local path relative to parent of created directory
+    if not user_input_config.startswith("/"):
+        test_path = Path("..") / user_input_config
+    else:
+        test_path = Path(user_input_config)
+
+    # check if user passed a local path
+    if test_path.exists() and test_path.is_dir():
+        local_path = test_path
+
+    elif test_path.exists() and test_path.endswith(".zip"):
+        tmp_zip = test_path
+
+    # check if user passed a url to a zip
+    elif user_input_config.startswith("http") and (
+        user_input_config.split(".")[-1] in ["zip"]
+    ):
+        tmp_zip, _ = urlretrieve(user_input_config)
+
+    # assume it is a VCS uri and try to clone
+    else:
+        clone(user_input_config, clone_to_dir=tmp)
+        local_path = tmp
+
+    if tmp_zip:
+        with ZipFile(tmp_zip, "r") as zipf:
+            zipf.extractall(tmp)
+            local_path = tmp
+
+    # write whatever the user supplied into the project
+    copy_tree(local_path, ".")
+
+    tmp.cleanup()

ccds/hook_utils/dependencies.py

@@ -0,0 +1,38 @@
+def write_dependencies(
+    dependencies, packages, pip_only_packages, repo_name, module_name, python_version
+):
+    if dependencies == "requirements.txt":
+        with open(dependencies, "w") as f:
+            lines = sorted(packages)
+
+            lines += ["" "-e ."]
+
+            f.write("\n".join(lines))
+            f.write("\n")
+
+    elif dependencies == "environment.yml":
+        with open(dependencies, "w") as f:
+            lines = [f"name: { repo_name }", "dependencies:"]
+
+            lines += [f"  - {p}" for p in packages if p not in pip_only_packages]
+
+            lines += ["  - pip:"] + [
+                f"    - {p}" for p in packages if p in pip_only_packages
+            ]
+
+            lines += ["    - -e ."]
+
+            lines += [f"  - python={python_version}"]
+
+            f.write("\n".join(lines))
+
+    elif dependencies == "Pipfile":
+        with open(dependencies, "w") as f:
+            lines = ["[packages]"]
+            lines += [f'{p} = "*"' for p in sorted(packages)]
+
+            lines += [f'"{ module_name }" ={{editable = true, path = "."}}']
+
+            lines += ["", "[requires]", f'python_version = "{ python_version }"']
+
+            f.write("\n".join(lines))