From ecbb00c30aa753b9d114c33a06042c5409c0e08e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 21 Feb 2024 09:37:43 -0600 Subject: [PATCH] Update event loop handling (#75) * Update event loop handling * add docstring * prefer_selector_loop * fix event policy handling on windows * another windows fix * update docstring * fix event loop handling * always prefer selector loop * simplify matrix * fix api usage * fix deps * close event loop after each test --- .github/workflows/downstream.yml | 24 +++++++------- .github/workflows/test.yml | 24 -------------- pyproject.toml | 2 +- pytest_jupyter/jupyter_client.py | 1 - pytest_jupyter/jupyter_core.py | 43 ++++++++++++------------- pytest_jupyter/jupyter_server.py | 36 ++------------------- pytest_jupyter/pytest_tornasync.py | 50 ++++++++++++++++-------------- tests/test_jupyter_server.py | 4 +++ 8 files changed, 67 insertions(+), 117 deletions(-) diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml index 46a4352..02aa1cd 100644 --- a/.github/workflows/downstream.yml +++ b/.github/workflows/downstream.yml @@ -10,29 +10,27 @@ concurrency: cancel-in-progress: true jobs: - jupyter_client: + jupyter_server: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Base Setup - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 - - - name: Run Test - uses: jupyterlab/maintainer-tools/.github/actions/downstream-test@v1 + - uses: actions/checkout@v4 + - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + - uses: jupyterlab/maintainer-tools/.github/actions/downstream-test@v1 with: - package_name: jupyter_client + package_name: jupyter_server + package_download_extra_args: --pre - jupyter_server: + jupyter_client: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 10 steps: - uses: actions/checkout@v4 - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 - uses: jupyterlab/maintainer-tools/.github/actions/downstream-test@v1 with: - package_name: jupyter_server + package_name: jupyter_client + package_download_extra_args: --pre downstream_check: # This job does nothing and is only used for the branch protection if: always() diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5d87d79..873a965 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -141,28 +141,6 @@ jobs: - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 - uses: jupyterlab/maintainer-tools/.github/actions/check-links@v1 - jupyter_server_downstream: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v4 - - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 - - uses: jupyterlab/maintainer-tools/.github/actions/downstream-test@v1 - with: - package_name: jupyter_server - package_download_extra_args: --pre - - jupyter_client_downstream: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v4 - - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 - - uses: jupyterlab/maintainer-tools/.github/actions/downstream-test@v1 - with: - package_name: jupyter_client - package_download_extra_args: --pre - tests_check: # This job does nothing and is only used for the branch protection if: always() needs: @@ -173,8 +151,6 @@ jobs: - test_prereleases - check_links - check_release - - jupyter_server_downstream - - jupyter_client_downstream - test_sdist runs-on: ubuntu-latest steps: diff --git a/pyproject.toml b/pyproject.toml index df4b065..456035d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ classifiers = [ ] dependencies = [ "pytest", - "jupyter_core" + "jupyter_core>=5.7" ] requires-python = ">=3.8" diff --git a/pytest_jupyter/jupyter_client.py b/pytest_jupyter/jupyter_client.py index c8b8e23..e3ebdd1 100644 --- a/pytest_jupyter/jupyter_client.py +++ b/pytest_jupyter/jupyter_client.py @@ -21,7 +21,6 @@ # Bring in local plugins. from pytest_jupyter.jupyter_core import * # noqa: F403 -from pytest_jupyter.pytest_tornasync import * # noqa: F403 @pytest.fixture() diff --git a/pytest_jupyter/jupyter_core.py b/pytest_jupyter/jupyter_core.py index 8186b1e..dd5da9e 100644 --- a/pytest_jupyter/jupyter_core.py +++ b/pytest_jupyter/jupyter_core.py @@ -1,15 +1,15 @@ """Fixtures for use with jupyter core and downstream.""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -import asyncio import json import os import sys -import typing +from inspect import iscoroutinefunction from pathlib import Path import jupyter_core import pytest +from jupyter_core.utils import ensure_event_loop from .utils import mkdir @@ -35,34 +35,35 @@ resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) -@pytest.fixture() +@pytest.fixture(autouse=True) def jp_asyncio_loop(): """Get an asyncio loop.""" - if os.name == "nt": - asyncio.set_event_loop_policy( - asyncio.WindowsSelectorEventLoopPolicy() # type:ignore[attr-defined] - ) - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = ensure_event_loop(prefer_selector_loop=True) yield loop loop.close() -@pytest.fixture(autouse=True) -def io_loop(jp_asyncio_loop): - """Override the io_loop for pytest_tornasync. This is a no-op - if tornado is not installed.""" +@pytest.hookimpl(tryfirst=True) +def pytest_pycollect_makeitem(collector, name, obj): + """Custom pytest collection hook.""" + if collector.funcnamefilter(name) and iscoroutinefunction(obj): + return list(collector._genfunctions(name, obj)) + return None + - async def get_tornado_loop() -> typing.Any: - """Asynchronously get a tornado loop.""" - try: - from tornado.ioloop import IOLoop +@pytest.hookimpl(tryfirst=True) +def pytest_pyfunc_call(pyfuncitem): + """Custom pytest function call hook.""" + funcargs = pyfuncitem.funcargs + testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames} - return IOLoop.current() - except ImportError: - pass + if not iscoroutinefunction(pyfuncitem.obj): + pyfuncitem.obj(**testargs) + return True - return jp_asyncio_loop.run_until_complete(get_tornado_loop()) + loop = ensure_event_loop(prefer_selector_loop=True) + loop.run_until_complete(pyfuncitem.obj(**testargs)) + return True @pytest.fixture() diff --git a/pytest_jupyter/jupyter_server.py b/pytest_jupyter/jupyter_server.py index a5c867d..c686577 100644 --- a/pytest_jupyter/jupyter_server.py +++ b/pytest_jupyter/jupyter_server.py @@ -3,7 +3,6 @@ # Distributed under the terms of the Modified BSD License. from __future__ import annotations -import asyncio import importlib import io import logging @@ -53,36 +52,6 @@ from pytest_jupyter.pytest_tornasync import * # noqa: F403 from pytest_jupyter.utils import mkdir -# Override some of the fixtures from pytest_tornasync -# The io_loop fixture is overridden in jupyter_core.py so it -# can be shared by other plugins that need it (e.g. jupyter_client.py). - - -@pytest.fixture() -def http_server(io_loop, http_server_port, jp_web_app): - """Start a tornado HTTP server that listens on all available interfaces.""" - - async def get_server(): - """Get a server asynchronously.""" - server = tornado.httpserver.HTTPServer(jp_web_app) - server.add_socket(http_server_port[0]) - return server - - server = io_loop.run_sync(get_server) - yield server - server.stop() - - if hasattr(server, "close_all_connections"): - try: - io_loop.run_sync(server.close_all_connections) - except asyncio.TimeoutError: - pass - - http_server_port[0].close() - - -# End pytest_tornasync overrides - @pytest.fixture() def jp_server_config(): @@ -177,7 +146,6 @@ def jp_configurable_serverapp( jp_root_dir, jp_logging_stream, jp_asyncio_loop, - io_loop, ): """Starts a Jupyter Server instance based on the provided configuration values. @@ -207,7 +175,6 @@ def _configurable_serverapp( environ=jp_environ, http_port=jp_http_port, tmp_path=tmp_path, - io_loop=io_loop, root_dir=jp_root_dir, **kwargs, ): @@ -345,7 +312,7 @@ async def my_test(jp_fetch, jp_ws_fetch): ... """ - def client_fetch(*parts, headers=None, params=None, **kwargs): # noqa: ARG + def client_fetch(*parts, headers=None, params=None, **kwargs): if not headers: headers = {} if not params: @@ -414,6 +381,7 @@ async def _(url, **fetch_kwargs): code = r.code except HTTPClientError as err: code = err.code + print(f"HTTPClientError ({err.code}): {err}") # noqa: T201 else: if fetch is jp_ws_fetch: r.close() diff --git a/pytest_jupyter/pytest_tornasync.py b/pytest_jupyter/pytest_tornasync.py index a789eb1..495ea9d 100644 --- a/pytest_jupyter/pytest_tornasync.py +++ b/pytest_jupyter/pytest_tornasync.py @@ -1,8 +1,8 @@ """Vendored fork of pytest_tornasync from https://github.com/eukaryote/pytest-tornasync/blob/9f1bdeec3eb5816e0183f975ca65b5f6f29fbfbb/src/pytest_tornasync/plugin.py """ +import asyncio from contextlib import closing -from inspect import iscoroutinefunction try: import tornado.ioloop @@ -14,33 +14,37 @@ import pytest # mypy: disable-error-code="no-untyped-call" +# Bring in local plugins. +from pytest_jupyter.jupyter_core import * # noqa: F403 -@pytest.hookimpl(tryfirst=True) -def pytest_pycollect_makeitem(collector, name, obj): - """Custom pytest collection hook.""" - if collector.funcnamefilter(name) and iscoroutinefunction(obj): - return list(collector._genfunctions(name, obj)) - return None +@pytest.fixture() +def io_loop(jp_asyncio_loop): + """Get the current tornado event loop.""" + return tornado.ioloop.IOLoop.current() + +@pytest.fixture() +def http_server(jp_asyncio_loop, http_server_port, jp_web_app): + """Start a tornado HTTP server that listens on all available interfaces.""" -@pytest.hookimpl(tryfirst=True) -def pytest_pyfunc_call(pyfuncitem): - """Custom pytest function call hook.""" - funcargs = pyfuncitem.funcargs - testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames} + async def get_server(): + """Get a server asynchronously.""" + server = tornado.httpserver.HTTPServer(jp_web_app) + server.add_socket(http_server_port[0]) + return server - if not iscoroutinefunction(pyfuncitem.obj): - pyfuncitem.obj(**testargs) - return True + server = jp_asyncio_loop.run_until_complete(get_server()) + yield server + server.stop() - try: - loop = funcargs["io_loop"] - except KeyError: - loop = tornado.ioloop.IOLoop.current() + if hasattr(server, "close_all_connections"): + try: + jp_asyncio_loop.run_until_complete(server.close_all_connections()) + except asyncio.TimeoutError: + pass - loop.run_sync(lambda: pyfuncitem.obj(**testargs)) - return True + http_server_port[0].close() @pytest.fixture() @@ -52,7 +56,7 @@ def http_server_port(): @pytest.fixture() -def http_server_client(http_server, io_loop): +def http_server_client(http_server, jp_asyncio_loop): """ Create an asynchronous HTTP client that can fetch from `http_server`. """ @@ -61,7 +65,7 @@ async def get_client(): """Get a client.""" return AsyncHTTPServerClient(http_server=http_server) - client = io_loop.run_sync(get_client) + client = jp_asyncio_loop.run_until_complete(get_client()) with closing(client) as context: yield context diff --git a/tests/test_jupyter_server.py b/tests/test_jupyter_server.py index 013ebaf..434609a 100644 --- a/tests/test_jupyter_server.py +++ b/tests/test_jupyter_server.py @@ -71,3 +71,7 @@ def test_template_dir(jp_template_dir): def test_extension_environ(jp_extension_environ): pass + + +def test_ioloop_fixture(io_loop): + pass