Merge pull request #52 from darrenburns/fixture-scoping

Test/module/global fixture scopes

README.md

@@ -84,11 +84,6 @@ def _(cities=city_list):
 
 Fixtures can be injected into each other, using the same syntax.
 
-The fixture will be executed exactly once each time a test depends on it. 
-
-More specifically, if a fixture F is required by multiple other fixtures that are all injected into a single
-test, then F will only be resolved once.
-
 Fixtures are great for extracting common setup code that you'd otherwise need to repeat at the top of your tests, 
 but they can also execute teardown code:
 
@@ -110,8 +105,17 @@ def _(db=database):
     expect(users).contains("Bob")
 ```
 
-The code below the `yield` statement in a fixture will be executed after the test that depends on it completes,
-regardless of the result of the test. 
+The code below the `yield` statement in the fixture will be executed after the test that depends on it completes,
+regardless of the result of the test.
+
+By default, a fixture will be executed exactly once each time a test depends on it.
+This is because the default `scope` of a fixture is `"test"`.
+
+More specifically, if a test-scoped fixture F is required by multiple other fixtures that are all injected into a single
+test, then F will only be resolved once.
+
+You can alter the scope of a fixture using the `scope` argument of the `fixture` decorator.
+For example `@fixture(scope="module")` will tell Ward to only execute the fixture at most once per module, and have all other tests in the module use the cached value from this execution.
 
 ### Descriptive testing
 

setup.py

@@ -2,7 +2,7 @@
 
 from setuptools import setup
 
-version = "0.13.0a0"
+version = "0.14.0a0"
 description = "A modern Python 3 test framework for finding and fixing flaws faster."
 with open("README.md", "r") as fh:
     if platform.system() != "Windows":

tests/test_suite.py

@@ -2,10 +2,10 @@
 
 from ward import expect, fixture
 from ward.fixtures import Fixture
-from ward.models import SkipMarker
+from ward.models import Scope, SkipMarker
 from ward.suite import Suite
 from ward.test_result import TestOutcome, TestResult
-from ward.testing import Test, test
+from ward.testing import Test, skip, test
 
 NUMBER_OF_TESTS = 5
 
@@ -33,10 +33,7 @@ def a(b=b):
 
 @fixture
 def fixtures(a=fixture_a, b=fixture_b):
-    return {
-        "fixture_a": Fixture(fn=a),
-        "fixture_b": Fixture(fn=b),
-    }
+    return {"fixture_a": Fixture(fn=a), "fixture_b": Fixture(fn=b)}
 
 
 @fixture
@@ -206,3 +203,159 @@ def test(b=b, c=c):
     list(suite.generate_test_runs())
 
     expect(events).equals([1, 2, 3])
+
+
+@test("Suite.generate_test_runs correctly tears down module scoped fixtures")
+def _():
+    events = []
+
+    @fixture(scope=Scope.Module)
+    def a():
+        events.append("resolve")
+        yield "a"
+        events.append("teardown")
+
+    def test1(a=a):
+        events.append("test1")
+
+    def test2(a=a):
+        events.append("test2")
+
+    def test3(a=a):
+        events.append("test3")
+
+    suite = Suite(
+        tests=[
+            Test(fn=test1, module_name="module1"),
+            Test(fn=test2, module_name="module2"),
+            Test(fn=test3, module_name="module2"),
+        ]
+    )
+
+    list(suite.generate_test_runs())
+
+    expect(events).equals(
+        [
+            "resolve",  # Resolve at start of module1
+            "test1",
+            "teardown",  # Teardown at end of module1
+            "resolve",  # Resolve at start of module2
+            "test2",
+            "test3",
+            "teardown",  # Teardown at end of module2
+        ]
+    )
+    expect(len(suite.cache)).equals(0)
+
+
+@test("Suite.generate_test_runs resolves and tears down global fixtures once only")
+def _():
+    events = []
+
+    @fixture(scope=Scope.Global)
+    def a():
+        events.append("resolve")
+        yield "a"
+        events.append("teardown")
+
+    def test1(a=a):
+        events.append("test1")
+
+    def test2(a=a):
+        events.append("test2")
+
+    def test3(a=a):
+        events.append("test3")
+
+    suite = Suite(
+        tests=[
+            Test(fn=test1, module_name="module1"),
+            Test(fn=test2, module_name="module2"),
+            Test(fn=test3, module_name="module2"),
+        ]
+    )
+
+    list(suite.generate_test_runs())
+
+    expect(events).equals(
+        [
+            "resolve",  # Resolve at start of run only
+            "test1",
+            "test2",
+            "test3",
+            "teardown",  # Teardown only at end of run
+        ]
+    )
+    expect(len(suite.cache)).equals(0)  # Teardown includes cache cleanup
+
+
+@test("Suite.generate_test_runs resolves mixed scope fixtures correctly")
+def _():
+    events = []
+
+    @fixture(scope=Scope.Global)
+    def a():
+        events.append("resolve a")
+        yield "a"
+        events.append("teardown a")
+
+    @fixture(scope=Scope.Module)
+    def b():
+        events.append("resolve b")
+        yield "b"
+        events.append("teardown b")
+
+    @fixture(scope=Scope.Test)
+    def c():
+        events.append("resolve c")
+        yield "c"
+        events.append("teardown c")
+
+    def test1(a=a, b=b, c=c):
+        events.append("test1")
+
+    def test2(a=a, b=b, c=c):
+        events.append("test2")
+
+    def test3(a=a, b=b, c=c):
+        events.append("test3")
+
+    suite = Suite(
+        tests=[
+            Test(fn=test1, module_name="module1"),
+            Test(fn=test2, module_name="module2"),
+            Test(fn=test3, module_name="module2"),
+        ]
+    )
+
+    list(suite.generate_test_runs())
+
+    # Note that the ordering of the final teardowns aren't well-defined
+    expect(events).equals(
+        [
+            "resolve a",  # global fixture so resolved at start
+            "resolve b",  # module fixture resolved at start of module1
+            "resolve c",  # test fixture resolved at start of test1
+            "test1",
+            "teardown c",  # test fixture teardown at start of test1
+            "teardown b",  # module fixture teardown at end of module1
+            "resolve b",  # module fixture resolved at start of module2
+            "resolve c",  # test fixture resolved at start of test2
+            "test2",
+            "teardown c",  # test fixture teardown at start of test2
+            "resolve c",  # test fixture resolved at start of test3
+            "test3",
+            "teardown c",  # test fixture teardown at end of test3
+            "teardown a",  # global fixtures are torn down at the very end
+            "teardown b",  # module fixture teardown at end of module2
+        ]
+    )
+    expect(len(suite.cache)).equals(0)
+
+
+@skip("WIP")
+@test(
+    "Suite.generate_test_runs dependent fixtures of differing scopes behave correctly"
+)
+def _():
+    pass

ward/__init__.py

@@ -1,3 +1,4 @@
 from .expect import expect, raises
 from .fixtures import fixture
 from .testing import skip, xfail, test
+from .models import Scope

ward/errors.py

@@ -0,0 +1,2 @@
+class FixtureError(Exception):
+    pass

ward/expect.py

@@ -158,7 +158,7 @@ def _store_in_history(
         result: bool,
         called_with_args: Tuple[Any],
         called_with_kwargs: Dict[str, Any],
-        that=None,
+        that: Any = None,
     ) -> bool:
         self.history.append(
             Expected(

ward/fixtures.py

@@ -2,36 +2,35 @@
 from contextlib import suppress
 from dataclasses import dataclass, field
 from functools import partial, wraps
-from typing import Callable, Dict
+from typing import Callable, Dict, Union, Optional, List
 
-from ward.models import WardMeta
-
-
-class TestSetupError(Exception):
-    pass
-
-
-class CollectionError(TestSetupError):
-    pass
-
-
-class FixtureExecutionError(Exception):
-    pass
+from ward.models import WardMeta, Scope
 
 
 @dataclass
 class Fixture:
-    def __init__(self, fn: Callable):
+    def __init__(
+        self,
+        fn: Callable,
+        last_resolved_module_name: Optional[str] = None,
+        last_resolved_test_id: Optional[str] = None,
+    ):
         self.fn = fn
         self.gen = None
         self.resolved_val = None
+        self.last_resolved_module_name = last_resolved_module_name
+        self.last_resolved_test_id = last_resolved_test_id
 
     @property
-    def key(self):
+    def key(self) -> str:
         path = inspect.getfile(fixture)
         name = self.name
         return f"{path}::{name}"
 
+    @property
+    def scope(self) -> Scope:
+        return getattr(self.fn, "ward_meta").scope
+
     @property
     def name(self):
         return self.fn.__name__
@@ -44,8 +43,11 @@ def deps(self):
         return inspect.signature(self.fn).parameters
 
     def teardown(self):
-        if self.is_generator_fixture:
-            next(self.gen)
+        # Suppress because we can't know whether there's more code
+        # to execute below the yield.
+        with suppress(StopIteration, RuntimeError):
+            if self.is_generator_fixture:
+                next(self.gen)
 
 
 @dataclass
@@ -57,28 +59,60 @@ def cache_fixture(self, fixture: Fixture):
 
     def teardown_all(self):
         """Run the teardown code for all generator fixtures in the cache"""
-        for fixture in self._fixtures.values():
+        vals = [f for f in self._fixtures.values()]
+        for fixture in vals:
             with suppress(RuntimeError, StopIteration):
                 fixture.teardown()
-
-    def __contains__(self, key: str):
+                del self[fixture.key]
+
+    def get(
+        self, scope: Optional[Scope], module_name: Optional[str], test_id: Optional[str]
+    ) -> List[Fixture]:
+        filtered_by_mod = [
+            f
+            for f in self._fixtures.values()
+            if f.scope == scope and f.last_resolved_module_name == module_name
+        ]
+
+        if test_id:
+            return [f for f in filtered_by_mod if f.last_resolved_test_id == test_id]
+        else:
+            return filtered_by_mod
+
+    def teardown_fixtures(self, fixtures: List[Fixture]):
+        for fixture in fixtures:
+            if fixture.key in self:
+                with suppress(RuntimeError, StopIteration):
+                    fixture.teardown()
+                    del self[fixture.key]
+
+    def __contains__(self, key: str) -> bool:
         return key in self._fixtures
 
-    def __getitem__(self, item):
-        return self._fixtures[item]
+    def __getitem__(self, key: str) -> Fixture:
+        return self._fixtures[key]
+
+    def __delitem__(self, key: str):
+        del self._fixtures[key]
+
+    def __len__(self):
+        return len(self._fixtures)
+
 
+def fixture(func=None, *, scope: Optional[Union[Scope, str]] = Scope.Test):
+    if not isinstance(scope, Scope):
+        scope = Scope.from_str(scope)
 
-def fixture(func=None, *, description=None):
     if func is None:
-        return partial(fixture, description=description)
+        return partial(fixture, scope=scope)
 
     # By setting is_fixture = True, the framework will know
     # that if this fixture is provided as a default arg, it
     # is responsible for resolving the value.
     if hasattr(func, "ward_meta"):
         func.ward_meta.is_fixture = True
     else:
-        func.ward_meta = WardMeta(is_fixture=True)
+        func.ward_meta = WardMeta(is_fixture=True, scope=scope)
 
     @wraps(func)
     def wrapper(*args, **kwargs):