From 33bb72c4532facc7afdcb3735d2f23bd7b814130 Mon Sep 17 00:00:00 2001 From: David Brochart Date: Thu, 31 Aug 2023 17:43:51 +0200 Subject: [PATCH] Use anyio instead of asyncio --- .devcontainer/requirements.txt | 1 - jupyverse_api/jupyverse_api/contents/__init__.py | 6 +++--- jupyverse_api/pyproject.toml | 1 + .../fps_auth_jupyterhub/routes.py | 3 ++- plugins/auth_jupyterhub/pyproject.toml | 1 + plugins/contents/fps_contents/fileid.py | 16 ++++++++-------- plugins/terminals/fps_terminals/server.py | 3 ++- plugins/terminals/fps_terminals/win_server.py | 3 ++- plugins/terminals/pyproject.toml | 1 + plugins/webdav/pyproject.toml | 1 - plugins/webdav/tests/test_webdav.py | 2 +- plugins/yjs/fps_yjs/routes.py | 9 +++++---- pyproject.toml | 2 +- tests/test_app.py | 2 +- tests/test_auth.py | 12 ++++++------ tests/test_contents.py | 2 +- tests/test_kernels.py | 2 +- tests/test_server.py | 8 ++++---- tests/test_settings.py | 2 +- 19 files changed, 41 insertions(+), 36 deletions(-) diff --git a/.devcontainer/requirements.txt b/.devcontainer/requirements.txt index 80223ac3..7ef1a5de 100644 --- a/.devcontainer/requirements.txt +++ b/.devcontainer/requirements.txt @@ -10,4 +10,3 @@ fps[unicorn] >=0.0.10 mypy pytest -pytest-asyncio diff --git a/jupyverse_api/jupyverse_api/contents/__init__.py b/jupyverse_api/jupyverse_api/contents/__init__.py index 5f9b018f..1b6b890c 100644 --- a/jupyverse_api/jupyverse_api/contents/__init__.py +++ b/jupyverse_api/jupyverse_api/contents/__init__.py @@ -1,8 +1,8 @@ -import asyncio from abc import ABC, abstractmethod from pathlib import Path from typing import Dict, List, Optional, Union +from anyio import Event from fastapi import APIRouter, Depends, Request, Response from jupyverse_api import Router @@ -12,8 +12,8 @@ class FileIdManager(ABC): - stop_watching_files: asyncio.Event - stopped_watching_files: asyncio.Event + stop_watching_files: Event + stopped_watching_files: Event @abstractmethod async def get_path(self, file_id: str) -> str: diff --git a/jupyverse_api/pyproject.toml b/jupyverse_api/pyproject.toml index bb4d0e43..7e7f8b95 100644 --- a/jupyverse_api/pyproject.toml +++ b/jupyverse_api/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "rich-click >=1.6.1,<2", "asphalt >=4.11.0,<5", "asphalt-web[fastapi] >=1.1.0,<2", + "anyio>=3.6.2,<5", ] dynamic = ["version"] diff --git a/plugins/auth_jupyterhub/fps_auth_jupyterhub/routes.py b/plugins/auth_jupyterhub/fps_auth_jupyterhub/routes.py index 9e835384..5c8b5a22 100644 --- a/plugins/auth_jupyterhub/fps_auth_jupyterhub/routes.py +++ b/plugins/auth_jupyterhub/fps_auth_jupyterhub/routes.py @@ -8,6 +8,7 @@ from typing_extensions import Annotated import httpx +from anyio import Lock from fastapi import APIRouter, Cookie, Depends, HTTPException, Request, WebSocket, status from fastapi.responses import RedirectResponse from jupyterhub.services.auth import HubOAuth @@ -31,7 +32,7 @@ class AuthJupyterHub(Auth, Router): def __init__(self) -> None: super().__init__(app) self.hub_auth = HubOAuth() - self.db_lock = asyncio.Lock() + self.db_lock = Lock() self.activity_url = os.environ.get("JUPYTERHUB_ACTIVITY_URL") self.server_name = os.environ.get("JUPYTERHUB_SERVER_NAME") self.background_tasks = set() diff --git a/plugins/auth_jupyterhub/pyproject.toml b/plugins/auth_jupyterhub/pyproject.toml index 62b5037b..83a93375 100644 --- a/plugins/auth_jupyterhub/pyproject.toml +++ b/plugins/auth_jupyterhub/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "httpx >=0.24.1,<1", "jupyterhub >=4.0.1,<5", "jupyverse-api >=0.1.2,<1", + "anyio>=3.6.2,<5", ] [[project.authors]] diff --git a/plugins/contents/fps_contents/fileid.py b/plugins/contents/fps_contents/fileid.py index 1b251361..0ecdcee3 100644 --- a/plugins/contents/fps_contents/fileid.py +++ b/plugins/contents/fps_contents/fileid.py @@ -4,7 +4,7 @@ from uuid import uuid4 import aiosqlite -from anyio import Path +from anyio import Event, Lock, Path from jupyverse_api import Singleton from watchfiles import Change, awatch @@ -14,7 +14,7 @@ class Watcher: def __init__(self, path: str) -> None: self.path = path - self._event = asyncio.Event() + self._event = Event() def __aiter__(self): return self @@ -31,18 +31,18 @@ def notify(self, change): class FileIdManager(metaclass=Singleton): db_path: str - initialized: asyncio.Event + initialized: Event watchers: Dict[str, List[Watcher]] - lock: asyncio.Lock + lock: Lock def __init__(self, db_path: str = ".fileid.db"): self.db_path = db_path - self.initialized = asyncio.Event() + self.initialized = Event() self.watchers = {} self.watch_files_task = asyncio.create_task(self.watch_files()) - self.stop_watching_files = asyncio.Event() - self.stopped_watching_files = asyncio.Event() - self.lock = asyncio.Lock() + self.stop_watching_files = Event() + self.stopped_watching_files = Event() + self.lock = Lock() async def get_id(self, path: str) -> Optional[str]: await self.initialized.wait() diff --git a/plugins/terminals/fps_terminals/server.py b/plugins/terminals/fps_terminals/server.py index 331c3fed..8aff9f64 100644 --- a/plugins/terminals/fps_terminals/server.py +++ b/plugins/terminals/fps_terminals/server.py @@ -6,6 +6,7 @@ import struct import termios +from anyio import Event from fastapi import WebSocketDisconnect from jupyverse_api.terminals import TerminalServer @@ -29,7 +30,7 @@ def __init__(self): async def serve(self, websocket, permissions): self.websocket = websocket self.websockets.append(websocket) - self.event = asyncio.Event() + self.event = Event() self.loop = asyncio.get_event_loop() task = asyncio.create_task(self.send_data()) diff --git a/plugins/terminals/fps_terminals/win_server.py b/plugins/terminals/fps_terminals/win_server.py index 9aa3e259..af9610ff 100644 --- a/plugins/terminals/fps_terminals/win_server.py +++ b/plugins/terminals/fps_terminals/win_server.py @@ -1,6 +1,7 @@ import asyncio import os +from anyio import sleep from jupyverse_api.terminals import TerminalServer from winpty import PTY # type: ignore @@ -36,7 +37,7 @@ async def send_data(self): await self.websocket.send_json(["disconnect", 1]) return if not data: - await asyncio.sleep(0.1) + await sleep(0.1) else: for websocket in self.websockets: await websocket.send_json(["stdout", data]) diff --git a/plugins/terminals/pyproject.toml b/plugins/terminals/pyproject.toml index 2273d1b8..65fff574 100644 --- a/plugins/terminals/pyproject.toml +++ b/plugins/terminals/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "websockets", "pywinpty;platform_system=='Windows'", "jupyverse-api >=0.1.2,<1", + "anyio>=3.6.2,<5", ] dynamic = ["version"] [[project.authors]] diff --git a/plugins/webdav/pyproject.toml b/plugins/webdav/pyproject.toml index 718adeee..62f23725 100644 --- a/plugins/webdav/pyproject.toml +++ b/plugins/webdav/pyproject.toml @@ -30,7 +30,6 @@ Homepage = "https://jupyter.org" test = [ "easywebdav", "pytest", - "pytest-asyncio", ] [tool.check-manifest] diff --git a/plugins/webdav/tests/test_webdav.py b/plugins/webdav/tests/test_webdav.py index 9ec5d82e..c309f434 100644 --- a/plugins/webdav/tests/test_webdav.py +++ b/plugins/webdav/tests/test_webdav.py @@ -24,7 +24,7 @@ def configure(components, config): return _components -@pytest.mark.asyncio +@pytest.mark.anyio @pytest.mark.skipif(sys.version_info < (3, 10), reason="Requires Python >=3.10") async def test_webdav(unused_tcp_port): components = configure( diff --git a/plugins/yjs/fps_yjs/routes.py b/plugins/yjs/fps_yjs/routes.py index 148c5045..8e263ae6 100644 --- a/plugins/yjs/fps_yjs/routes.py +++ b/plugins/yjs/fps_yjs/routes.py @@ -7,6 +7,7 @@ from typing import Dict from uuid import uuid4 +from anyio import Lock, sleep from fastapi import ( HTTPException, Request, @@ -138,7 +139,7 @@ class RoomManager: cleaners: Dict[YRoom, asyncio.Task] last_modified: Dict[str, datetime] websocket_server: JupyterWebsocketServer - lock: asyncio.Lock + lock: Lock def __init__(self, contents: Contents): self.contents = contents @@ -149,7 +150,7 @@ def __init__(self, contents: Contents): self.last_modified = {} # a dictionary of file_id:last_modification_date self.websocket_server = JupyterWebsocketServer(rooms_ready=False, auto_clean_rooms=False) self.websocket_server_task = asyncio.create_task(self.websocket_server.start()) - self.lock = asyncio.Lock() + self.lock = Lock() def stop(self): for watcher in self.watchers.values(): @@ -305,7 +306,7 @@ async def maybe_save_document( self, file_id: str, file_type: str, file_format: str, document: YBaseDoc ) -> None: # save after 1 second of inactivity to prevent too frequent saving - await asyncio.sleep(1) # FIXME: pass in config + await sleep(1) # FIXME: pass in config # if the room cannot be found, don't save try: file_path = await self.get_file_path(file_id, document) @@ -342,7 +343,7 @@ async def maybe_save_document( async def maybe_clean_room(self, room, ws_path: str) -> None: file_id = ws_path.split(":", 2)[2] # keep the document for a while in case someone reconnects - await asyncio.sleep(60) # FIXME: pass in config + await sleep(60) # FIXME: pass in config document = self.documents[ws_path] document.unobserve() del self.documents[ws_path] diff --git a/pyproject.toml b/pyproject.toml index af2dff7b..235da01a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,8 +43,8 @@ noauth = ["fps-noauth >=0.1.2,<1"] test = [ "mypy", "types-setuptools", + "anyio>=3.6.2,<5", "pytest", - "pytest-asyncio", "pytest-rerunfailures", "pytest-timeout", "pytest-env", diff --git a/tests/test_app.py b/tests/test_app.py index d2516a5a..ac77ccaa 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -9,7 +9,7 @@ from utils import configure -@pytest.mark.asyncio +@pytest.mark.anyio @pytest.mark.parametrize( "mount_path", ( diff --git a/tests/test_auth.py b/tests/test_auth.py index 4f76d2a9..74feb765 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -20,7 +20,7 @@ } -@pytest.mark.asyncio +@pytest.mark.anyio async def test_kernel_channels_unauthenticated(unused_tcp_port): async with Context() as ctx: await JupyverseComponent( @@ -35,7 +35,7 @@ async def test_kernel_channels_unauthenticated(unused_tcp_port): pass -@pytest.mark.asyncio +@pytest.mark.anyio async def test_kernel_channels_authenticated(unused_tcp_port): async with Context() as ctx, AsyncClient() as http: await JupyverseComponent( @@ -51,7 +51,7 @@ async def test_kernel_channels_authenticated(unused_tcp_port): pass -@pytest.mark.asyncio +@pytest.mark.anyio @pytest.mark.parametrize("auth_mode", ("noauth", "token", "user")) async def test_root_auth(auth_mode, unused_tcp_port): components = configure(COMPONENTS, {"auth": {"mode": auth_mode}}) @@ -71,7 +71,7 @@ async def test_root_auth(auth_mode, unused_tcp_port): assert response.headers["content-type"] == "application/json" -@pytest.mark.asyncio +@pytest.mark.anyio @pytest.mark.parametrize("auth_mode", ("noauth",)) async def test_no_auth(auth_mode, unused_tcp_port): components = configure(COMPONENTS, {"auth": {"mode": auth_mode}}) @@ -85,7 +85,7 @@ async def test_no_auth(auth_mode, unused_tcp_port): assert response.status_code == 200 -@pytest.mark.asyncio +@pytest.mark.anyio @pytest.mark.parametrize("auth_mode", ("token",)) async def test_token_auth(auth_mode, unused_tcp_port): components = configure(COMPONENTS, {"auth": {"mode": auth_mode}}) @@ -105,7 +105,7 @@ async def test_token_auth(auth_mode, unused_tcp_port): assert response.status_code == 302 -@pytest.mark.asyncio +@pytest.mark.anyio @pytest.mark.parametrize("auth_mode", ("user",)) @pytest.mark.parametrize( "permissions", diff --git a/tests/test_contents.py b/tests/test_contents.py index c74eab1c..ec1d6d9f 100644 --- a/tests/test_contents.py +++ b/tests/test_contents.py @@ -16,7 +16,7 @@ } -@pytest.mark.asyncio +@pytest.mark.anyio @pytest.mark.parametrize("auth_mode", ("noauth",)) async def test_tree(auth_mode, tmp_path, unused_tcp_port): prev_dir = os.getcwd() diff --git a/tests/test_kernels.py b/tests/test_kernels.py index 863504a7..009b12c0 100644 --- a/tests/test_kernels.py +++ b/tests/test_kernels.py @@ -26,7 +26,7 @@ } -@pytest.mark.asyncio +@pytest.mark.anyio @pytest.mark.parametrize("auth_mode", ("noauth",)) async def test_kernel_messages(auth_mode, capfd, unused_tcp_port): kernel_id = "kernel_id_0" diff --git a/tests/test_server.py b/tests/test_server.py index 07993dbc..6c6f75a2 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -1,10 +1,10 @@ -import asyncio import json from pathlib import Path import pytest import requests import y_py as Y +from anyio import sleep from websockets import connect from ypy_websocket import WebsocketProvider @@ -45,7 +45,7 @@ def test_settings_persistence_get(start_jupyverse): assert response.status_code == 204 -@pytest.mark.asyncio +@pytest.mark.anyio @pytest.mark.parametrize("auth_mode", ("noauth",)) @pytest.mark.parametrize("clear_users", (False,)) async def test_rest_api(start_jupyverse): @@ -85,7 +85,7 @@ async def test_rest_api(start_jupyverse): ) as websocket, WebsocketProvider(ydoc, websocket): # connect to the shared notebook document # wait for file to be loaded and Y model to be created in server and client - await asyncio.sleep(0.5) + await sleep(0.5) # execute notebook for cell_idx in range(3): response = requests.post( @@ -98,7 +98,7 @@ async def test_rest_api(start_jupyverse): ), ) # wait for Y model to be updated - await asyncio.sleep(0.5) + await sleep(0.5) # retrieve cells cells = json.loads(ydoc.get_array("cells").to_json()) assert cells[0]["outputs"] == [ diff --git a/tests/test_settings.py b/tests/test_settings.py index 743276f5..29b81882 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -22,7 +22,7 @@ } -@pytest.mark.asyncio +@pytest.mark.anyio @pytest.mark.parametrize("auth_mode", ("noauth",)) async def test_settings(auth_mode, unused_tcp_port): components = configure(COMPONENTS, {"auth": {"mode": auth_mode}})