[WIP] Refactoring Ocean Tools (#58)

* added Ocean tools. Removed all traces of oceanbench. Added new ocn-tools dependency. * Removed more ocean tools. * Removed rogue add units. * Slimmed down the test environment.yaml * refactored mamba test environment. * fixed yaml. further refactor. * Try poetry test framework. * updated pyproject. removed cmake. * update. * fix again. * added linux dist this time. * added dummy test data. * Updated environment with ocn-tools.
jejjohnson · Oct 9, 2023 · 5d3de4c · 5d3de4c
1 parent d2d9deb
commit 5d3de4c
Show file tree

Hide file tree

Showing 50 changed files with 3,988 additions and 5,285 deletions.
diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
@@ -1,20 +1,26 @@
-defaults:
-  run:
-    shell: bash -l {0}
-name: PyTest
-on: [pull_request]
+name: Run Tests
+on:
+  pull_request:
+  push:
+    branches:
+      - main
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-
-      - name: Install Conda environment with Micromamba
-        uses: mamba-org/provision-with-micromamba@main
+      - name: Check out the code
+        uses: actions/[email protected]
+      - name: Install Micromamba
+        uses: mamba-org/setup-micromamba@v1
         with:
+          micromamba-version: "1.4.5-0"
           environment-file: environments/linux.yaml
-
+          init-shell: bash
+          cache-environment: true
+          post-cleanup: "all"
       - name: install xarray-dataclasses
         run: pip install --no-deps xarray-dataclasses
+        shell: micromamba-shell {0}
       - name: pytest
         run: pytest oceanbench/_src
+        shell: micromamba-shell {0}
diff --git a/.gitignore b/.gitignore
@@ -156,3 +156,5 @@ notebooks/*.gif
 .DS_Store
 .vscode/settings.json
 \jbook/_build/
+
+\oceanbench-data-registry/
diff --git a/README.md b/README.md
@@ -48,6 +48,13 @@ Some highlighted tutorials are listed in the next section.
 We have an open data registry located at the [oceanbench-data-registry](https://github.com/quentinf00/oceanbench-data-registry) GitHub repository.
 You can find some more meta-data about the available datasets as well as how to download them yourself.
 
+**Ocean-Tools** [![Open In GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/jejjohnson/ocn-tools)
+
+We have our utility functions which make up the backbone of the preprocessing, postprocessing, and some plotting. 
+This is what we piece together to use for creating recipes, pipelines and tasks.
+This repo is located at [jejjohnson/ocn-tools](https://github.com/jejjohnson/ocn-tools). 
+See the docs (**TODO**) for more information.
+
 **Machine Learning Datasets** [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://jejjohnson.github.io/oceanbench/content/getting_started/TaskToPatcher.html)
 
 We have a set of tasks related to sea surface height interpolation and they come readily integrated into a digestable ML-ready format.

diff --git a/config/dataset/components/patcher.yaml b/config/dataset/components/patcher.yaml
@@ -6,9 +6,9 @@ strides: null
 ref_grid_var: null
 
 cfg:
-    _target_: oceanbench._src.datasets.base.XRDAPatcher
+    _target_: xrpatcher._src.base.XRDAPatcher
     da: 
-        _target_: oceanbench._src.utils.data.interpstack_dataarrays
+        _target_: ocn_tools._src.utils.data.interpstack_dataarrays
         dataarrays: '${...das}'
         ref_grid_var: '${...ref_grid_var}'
     domain_limits: 

diff --git a/config/leaderboard/ose_gf.yaml b/config/leaderboard/ose_gf.yaml
@@ -3,7 +3,7 @@ method: duacs
 
 outputs:
   eval_ds:
-    _target_: oceanbench._src.utils.data.stack_dataarrays
+    _target_: ocn_tools._src.utils.data.stack_dataarrays
     _partial_: true
     dataarrays: 
         ref: '${results.outputs.eval.ref}'
@@ -21,7 +21,7 @@ outputs:
 
   plot_ds:
     maps:
-      _target_: oceanbench._src.utils.data.stack_dataarrays
+      _target_: ocn_tools._src.utils.data.stack_dataarrays
       _partial_: true
       dataarrays: 
           ref: '${results.outputs.plots.maps.ref}'

diff --git a/config/leaderboard/osse_gf.yaml b/config/leaderboard/osse_gf.yaml
@@ -3,7 +3,7 @@ method: bfn
 
 outputs:
   eval_ds:
-    _target_: oceanbench._src.utils.data.stack_dataarrays
+    _target_: ocn_tools._src.utils.data.stack_dataarrays
     _partial_: true
     dataarrays: 
         ref: '${results.outputs.eval.ref}'
@@ -21,7 +21,7 @@ outputs:
 
   plot_ds:
     maps:
-      _target_: oceanbench._src.utils.data.stack_dataarrays
+      _target_: ocn_tools._src.utils.data.stack_dataarrays
       _partial_: true
       dataarrays: 
           ref: '${results.outputs.plots.maps.ref}'

diff --git a/config/metrics/components/alongtrack_psd_score.yaml b/config/metrics/components/alongtrack_psd_score.yaml
@@ -1,6 +1,6 @@
 
 alongtrack_segments:
-  _target_: "oceanbench._src.preprocessing.alongtrack.select_track_segments"
+  _target_: "ocn_tools._src.preprocessing.alongtrack.select_track_segments"
   _partial_: True
   variable: "ref"
   variable_interp: "study"
@@ -10,7 +10,7 @@ alongtrack_segments:
   segment_overlapping: 0.25
 
 psd_welch_score:
-  _target_: "oceanbench._src.metrics.power_spectrum.psd_welch_score"
+  _target_: "ocn_tools._src.metrics.power_spectrum.psd_welch_score"
   _partial_: True
   variable: "study"
   variable_ref: "ref"

diff --git a/config/metrics/components/nrmse.yaml b/config/metrics/components/nrmse.yaml
@@ -2,7 +2,7 @@
 dim: ["lat", "lon", "time"]
 
 cfg:
-    _target_: "oceanbench._src.metrics.stats.nrmse_ds"
+    _target_: "ocn_tools._src.metrics.stats.nrmse_ds"
     _partial_: True
     target: study
     reference: ref

diff --git a/config/metrics/lib.yaml b/config/metrics/lib.yaml
@@ -1,7 +1,7 @@
 #@package _global_
 
 psd_isotropic_score:
-    _target_: "oceanbench._src.metrics.power_spectrum.psd_isotropic_score"
+    _target_: "ocn_tools._src.metrics.power_spectrum.psd_isotropic_score"
     _partial_: True
     ref_variable: ref
     study_variable: study
@@ -15,7 +15,7 @@ psd_isotropic_score:
     truncate: True
 
 psd_spacetime_score:
-    _target_: "oceanbench._src.metrics.power_spectrum.psd_spacetime_score"
+    _target_: "ocn_tools._src.metrics.power_spectrum.psd_spacetime_score"
     _partial_: True
     ref_variable: ref
     study_variable: study

diff --git a/config/plots/gfmaps.yaml b/config/plots/gfmaps.yaml
@@ -13,7 +13,7 @@ cmaps:
         - {_target_: operator.attrgetter, _args_: [speed]}
         - {_target_: holoviews.plotting.util.process_cmap, _partial_: True}
 
-geocalc: oceanbench._src.geoprocessing.geostrophic_v2
+geocalc: ocn_tools._src.geoprocessing.geostrophic_v2
 
 attrs:
   ke: {units: 'm$^{2}$s$^{-2}$', standard_name: kinetic_energy, long_name: Kinetic Energy}

diff --git a/config/plots/lib.yaml b/config/plots/lib.yaml
@@ -25,8 +25,8 @@ hook:
   _target_: oceanbench._src.utils.hydra.const_fn
   const: 
     inp: 
-    - {_target_: oceanbench._src.viz.ygrid_major, _partial_: True}
-    - {_target_: oceanbench._src.viz.xformatter, _partial_: True}
+    - {_target_: ocn_tools._src.viz.ygrid_major, _partial_: True}
+    - {_target_: ocn_tools._src.viz.xformatter, _partial_: True}
     _partial_: false
     fns: [{_target_: builtins.list, _partial_: true}]
 

diff --git a/config/processing/lib.yaml b/config/processing/lib.yaml
@@ -4,12 +4,12 @@ resample_method:  {target_: "xarray.Dataset.resample", _partial_: True, indexer:
 resample_aggregate: {_target_: "xarray.core.resample.DatasetResample.mean",  _partial_: True}
 
 fill_nans:
-    _target_: "oceanbench._src.geoprocessing.interpolate.fillnan_gauss_seidel"
+    _target_: "ocn_tools._src.geoprocessing.interpolate.fillnan_gauss_seidel"
     _partial_: True
     variable: "ssh"
 
 rescale_space:
-    _target_: "oceanbench._src.geoprocessing.spatial.latlon_deg2m"
+    _target_: "ocn_tools._src.geoprocessing.spatial.latlon_deg2m"
     _partial_: True
     mean: True
 
@@ -19,12 +19,12 @@ to_dataset:
     name: 'ssh'
 
 add_units:
-    _target_: oceanbench._src.data.add_units
+    _target_: ocn_tools._src.data.add_units
     _partial_: true
     units: {ssh: meter, lon: degrees_east, lat: degrees_north, time: seconds}
 
 psd_spacetime:
-    _target_: "oceanbench._src.metrics.power_spectrum.psd_spacetime"
+    _target_: "ocn_tools._src.metrics.power_spectrum.psd_spacetime"
     _partial_: True
     variable: "ssh"
     dims: ["time", "lon"]
@@ -36,7 +36,7 @@ psd_spacetime:
     truncate: True
 
 psd_isotropic:
-    _target_: "oceanbench._src.metrics.power_spectrum.psd_isotropic"
+    _target_: "ocn_tools._src.metrics.power_spectrum.psd_isotropic"
     _partial_: True
     variable: "ssh"
     dims: ["lon", "lat"]

diff --git a/config/processing/ose_gf_results.yaml b/config/processing/ose_gf_results.yaml
@@ -9,7 +9,7 @@ select:
   lon: '${task.outputs.domain.lon}'
 
 degrid:
-  _target_: oceanbench._src.geoprocessing.gridding.grid_to_coord_based
+  _target_: ocn_tools._src.geoprocessing.gridding.grid_to_coord_based
   _partial_: true
   tgt_coord_based_ds: {inp: '${...outputs.eval.ref}', fns: ['${F.call}'], _partial_: false}
 

diff --git a/environments/linux.yaml b/environments/linux.yaml
@@ -31,6 +31,7 @@ dependencies:
   - jupyter-book
   - ghp-import
   - pip:
+      - git+https://github.com/jejjohnson/ocn-tools.git
       - hydra-core
       - pyrootutils
       - loguru

diff --git a/environments/macos.yaml b/environments/macos.yaml
diff --git a/environments/test.yaml b/environments/test.yaml
@@ -0,0 +1,16 @@
+name: oceanbench
+channels:
+  - conda-forge
+  - pyviz
+dependencies:
+  - python=3.10
+  - xesmf
+  - pip
+  - pytest
+  - pip:
+      - git+https://github.com/jejjohnson/ocn-tools.git
+      - hydra-core
+      - pyrootutils
+      - loguru
+      - xrpatcher
+      - autoroot