Switch Omegaconf for cattrs, add config saving and reformat dependenc…

…y structure

pyproject.toml

@@ -28,16 +28,17 @@ classifiers = [
 dependencies = [
  "antlr4-python3-runtime==4.13.0",
  "attrs>=23.2.0",
+ "cattrs>=23.2.3",
  "case-converter>=1.1.0",
  "click>=8.1.7",
+ "DeepDiff>=6.7.1",
  "easyeda2ato>=0.1.5",
  "eseries>=1.2.1",
  "fastapi>=0.109.0",
  "gitpython>=3.1.41",
  "igraph>=0.11.3",
  "jinja2>=3.1.3",
  "natsort>=8.4.0",
- "omegaconf==2.4.0.dev1",
  "packaging>=23.2",
  "pandas>=2.1.4",
  "pint>=0.23",

src/atopile/cli/common.py

@@ -9,8 +9,6 @@
 from typing import Iterable
 
 import click
-from omegaconf import OmegaConf
-from omegaconf.errors import ConfigKeyError
 
 import atopile.config
 from atopile import address, errors, version
@@ -74,14 +72,12 @@ def wrapper(
 
  # add custom config overrides
  if option:
- try:
- config: atopile.config.ProjectConfig = OmegaConf.merge(
- project_config,
- OmegaConf.from_dotlist(option)
- )
- except ConfigKeyError as ex:
- raise click.BadParameter(f"Invalid config key {ex.key}") from ex
-
+ raise NotImplementedError(
+ "Custom config overrides have been removed in a refactor. "
+ "It's planned to re-add them in a future release. "
+ "If this is a blocker for you, please raise an issue. "
+ "In the meantime, you can use the `ato.yaml` file to set these options."
+ )
  else:
  config: atopile.config.ProjectConfig = project_config
 

src/atopile/config.py

@@ -2,21 +2,23 @@
 
 """Schema and utils for atopile config files."""
 
-import collections.abc
+import copy
 import fnmatch
 import logging
 from pathlib import Path
 from typing import Any, Optional
 
-import yaml
+import cattrs
+import deepdiff
+from ruamel.yaml import YAML
 from attrs import Factory, define
-from omegaconf import MISSING, OmegaConf
-from omegaconf.errors import ConfigKeyError
 
 import atopile.errors
+import atopile.version
 from atopile import address
 
 log = logging.getLogger(__name__)
+yaml = YAML()
 
 
 CONFIG_FILENAME = "ato.yaml"
@@ -25,6 +27,9 @@
 BUILD_DIR_NAME = "build"
 
 
+_converter = cattrs.Converter()
+
+
 @define
 class ProjectPaths:
  """Config grouping for all the paths in a project."""
@@ -37,7 +42,7 @@ class ProjectPaths:
 class ProjectBuildConfig:
  """Config for a build."""
 
- entry: str = MISSING
+ entry: Optional[str] = None
  targets: list[str] = ["__default__"]
 
 
@@ -47,78 +52,111 @@ class ProjectServicesConfig:
  components: str = "https://atopile-component-server-atsuhzfd5a-uc.a.run.app/jlc"
 
 
+@define
+class Dependency:
+ """A dependency for a project."""
+
+ name: str
+ version_spec: str = "^0.0.0"
+ path: Optional[Path] = None
+
+ @classmethod
+ def from_str(cls, spec_str: str) -> "Dependency":
+ """Create a Dependency object from a string."""
+ for splitter in atopile.version.OPERATORS + ("@",):
+ if splitter in spec_str:
+ try:
+ name, version_spec = spec_str.split(splitter)
+ except TypeError as ex:
+ raise atopile.errors.AtoTypeError(
+ f"Invalid dependency spec: {spec_str}"
+ ) from ex
+ return cls(name.strip(), version_spec.strip())
+ return cls(name=spec_str)
+
+
 @define
 class ProjectConfig:
  """
  The config object for atopile.
  """
 
- location: Path = MISSING
+ location: Optional[Path] = None
 
  ato_version: str = "0.1.0"
  paths: ProjectPaths = Factory(ProjectPaths)
  builds: dict[str, ProjectBuildConfig] = Factory(dict)
- dependencies: list[str] = []
+ dependencies: list[str | Dependency] = Factory(list)
  services: ProjectServicesConfig = Factory(ProjectServicesConfig)
 
+ @staticmethod
+ def sanitise_dict_keys(d: dict) -> dict:
+ """Sanitise the keys of a dictionary to be valid python identifiers."""
+ data = copy.deepcopy(d)
+ data["ato_version"] = data.pop("ato-version")
+ return data
 
-KEY_CONVERSIONS = {
- "ato-version": "ato_version",
-}
-
+ @staticmethod
+ def unsanitise_dict_keys(d: dict) -> dict:
+ """Sanitise the keys of a dictionary to be valid python identifiers."""
+ data = copy.deepcopy(d)
+ data["ato-version"] = data.pop("ato_version")
+ return data
 
-def _sanitise_key(key: str) -> str:
- """Sanitize a key."""
- return KEY_CONVERSIONS.get(key, key)
-
-
-def _sanitise_item(item: tuple[Any, Any]) -> tuple[Any, Any]:
- """Sanitise the key of a dictionary item to be a valid python identifier."""
- k, v = item
- if isinstance(v, collections.abc.Mapping):
- return _sanitise_key(k), _sanitise_dict_keys(v)
- return _sanitise_key(k), v
-
-
-def _sanitise_dict_keys(d: collections.abc.Mapping) -> collections.abc.Mapping:
- """Sanitise the keys of a dictionary to be valid python identifiers."""
- if d is None:
- return {}
- return dict(_sanitise_item(item) for item in d.items())
+ @classmethod
+ def structure(cls, data: dict) -> "ProjectConfig":
+ """Make a config object from a dictionary."""
+ return _converter.structure(cls.sanitise_dict_keys(data), cls)
+
+ def patch_config(self, original: dict) -> dict:
+ """Apply a delta between the original and the current config."""
+ original_cfg = self.structure(original)
+ delta = deepdiff.Delta(deepdiff.DeepDiff(
+ self.unsanitise_dict_keys(_converter.unstructure(original_cfg)),
+ self.unsanitise_dict_keys(_converter.unstructure(self)),
+ ))
+ return original + delta
+
+ def save_changes(
+ self,
+ location: Optional[Path] = None
+ ) -> None:
+ """
+ Save the changes to the config object
+ """
+ if location is None:
+ location = self.location
+
+ with location.open() as f:
+ original = yaml.load(f)
+
+ patched = self.patch_config(original)
+
+ with location.open("w") as f:
+ yaml.dump(patched, f)
 
+ @classmethod
+ def load(cls, location: Path) -> "ProjectConfig":
+ """
+ Make a config object for a project.
+ """
+ with location.open() as f:
+ config_data = yaml.load(f)
 
-def make_config(project_config: Path) -> ProjectConfig:
- """
- Make a config object for a project.
+ config = cls.structure(config_data)
+ config.location = location.parent.expanduser().resolve().absolute()
 
- The typing on this is a little white lie... because they're really OmegaConf objects.
- """
- structure: ProjectConfig = OmegaConf.structured(ProjectConfig())
+ return config
 
- with project_config.open() as f:
- config_data = yaml.safe_load(f)
- project_config_data = OmegaConf.create(_sanitise_dict_keys(config_data))
 
- structure.location = project_config.parent.expanduser().resolve().absolute()
+## Register hooks for cattrs to handle the custom types
 
- for _ in range(1000):
- try:
- return OmegaConf.merge(
- structure,
- project_config_data,
- )
- except ConfigKeyError as ex:
- dot_path = ex.full_key.split(".")
- container = project_config_data
- for key in dot_path[:-1]:
- container = container[key]
- del container[dot_path[-1]]
+_converter.register_structure_hook(
+ str | Dependency,
+ lambda d, _: Dependency.from_str(d) if isinstance(d, str) else d,
+)
 
- atopile.errors.AtoError(
- f"Unknown config option in {structure.location}. Ignoring \"{ex.full_key}\".",
- title="Unknown config option",
- ).log(log, logging.WARNING)
- raise atopile.errors.AtoError("Too many config errors")
+##
 
 
 def get_project_dir_from_path(path: Path) -> Path:
@@ -146,7 +184,7 @@ def get_project_config_from_path(path: Path) -> ProjectConfig:
  project_dir = get_project_dir_from_path(path)
  project_config_file = project_dir / CONFIG_FILENAME
  if project_config_file not in _loaded_configs:
- _loaded_configs[project_config_file] = make_config(project_config_file)
+ _loaded_configs[project_config_file] = ProjectConfig.load(project_config_file)
  return _loaded_configs[project_config_file]
 
 

tests/test_config.py

@@ -1,9 +1,30 @@
 from atopile import config
+from ruamel.yaml import YAML
+import copy
 
+yaml = YAML()
 
-def test_sanitise_dict_keys():
- """Test that dict keys are sanitised."""
- assert config._sanitise_dict_keys({"a-b": 1, "ato-version": {"e-f": 2}}) == {
- "a-b": 1,
- "ato_version": {"e-f": 2},
- }
+
+def test_roundtrip():
+ config_dict = yaml.load("""
+ ato-version: ^0.2.0
+ builds:
+ debug:
+ entry: elec/src/debug.ato:Debug
+ unknown: test
+ # comments
+ dependencies:
+ - tps63020dsjr # comments
+ - usb-connectors ^v2.0.1
+ - esp32-s3:
+ version: ^v0.0.1
+ path: ../esp32-s3
+ """)
+ cfg = config.ProjectConfig.structure(config_dict)
+ assert config_dict == cfg.patch_config(config_dict)
+ assert cfg.ato_version == "^0.2.0"
+
+ cfg.ato_version = "10"
+ config_dict_2 = copy.deepcopy(config_dict)
+ config_dict_2["ato-version"] = "10"
+ assert config_dict_2 == cfg.patch_config(config_dict)