Start uncythonization

.github/workflows/conda.yml

@@ -74,6 +74,4 @@ jobs:
         run: |
           # install anaconda for upload
           mamba install anaconda-client
-
           anaconda upload --label dev noarch/*.tar.bz2
-          anaconda upload --label dev linux-64/*.tar.bz2

.github/workflows/tests.yaml

@@ -78,14 +78,7 @@ jobs:
       - name: Install
         shell: bash -l {0}
         run: |
-          # Cythonize scheduler on Python 3.8 builds
-          if [[ "${{ matrix.python-version }}" = "3.8" ]]; then
-              python -m pip install -vv --no-deps --install-option="--with-cython" -e .
-              python -c "from distributed.scheduler import COMPILED; assert COMPILED"
-          else
-              python -m pip install --no-deps -e .
-              python -c "from distributed.scheduler import COMPILED; assert not COMPILED"
-          fi
+          python -m pip install --no-deps -e .
 
       - name: mamba list
         shell: bash -l {0}

continuous_integration/recipes/dask/meta.yaml

@@ -23,7 +23,7 @@ requirements:
   run:
     - python >=3.8
     - dask-core >={{ dask_version }}
-    - distributed {{ version }}=*{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}*
+    - distributed {{ version }}=*_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
     - cytoolz >=0.8.2
     - numpy >=1.18
     - pandas >=1.0

continuous_integration/recipes/distributed/conda_build_config.yaml

@@ -1,6 +1,3 @@
 python:
     - 3.8
     - 3.9
-cython_enabled:
-    - True   # [linux]
-    - False

continuous_integration/recipes/distributed/meta.yaml

@@ -2,109 +2,76 @@
 {% set new_patch = major_minor_patch[2] | int + 1 %}
 {% set version = (major_minor_patch[:2] + [new_patch]) | join('.') + environ.get('VERSION_SUFFIX', '') %}
 {% set dask_version = environ.get('DASK_CORE_VERSION', '0.0.0.dev') %}
-{% set build_ext = "cython" %}  # [cython_enabled]
-{% set build_ext = "python" %}  # [not cython_enabled]
 
 
 package:
-  name: distributed-split
+  name: distributed
   version: {{ version }}
 
 source:
   git_url: ../../..
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}
+  string: py_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+  noarch: python
+  script: python -m pip install . -vv --no-deps
+  entry_points:
+    - dask-scheduler = distributed.cli.dask_scheduler:main
+    - dask-ssh = distributed.cli.dask_ssh:main
+    - dask-worker = distributed.cli.dask_worker:main
 
-outputs:
-  - name: distributed-impl
-    version: {{ version }}
-    build:
-      number: {{ GIT_DESCRIBE_NUMBER }}
-      string: {{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}_{{ build_ext }}
-      noarch: generic
-    test:
-      commands:
-        - exit 0
-    about:
-      home: https://distributed.dask.org
-      summary: A meta-package to select Distributed pre-release build variant
-      license: BSD-3-Clause
-      license_family: BSD
-      license_file: LICENSE.txt
+requirements:
+  host:
+    - python >=3.8
+    - pip
+    - setuptools
+  run:
+    - python >=3.8
+    - click >=6.6
+    - cloudpickle >=1.5.0
+    - cytoolz >=0.8.2
+    - dask-core >={{ dask_version }}
+    - jinja2
+    - msgpack-python >=0.6.0
+    - packaging >=20.0
+    - psutil >=5.0
+    - pyyaml
+    - sortedcontainers !=2.0.0,!=2.0.1
+    - tblib >=1.6.0
+    - toolz >=0.8.2
+    - tornado >=6.0.3
+    - urllib3
+    - zict >=0.1.3
+  run_constrained:
+    - openssl !=1.1.1e
 
-  - name: distributed
-    version: {{ version }}
-    build:
-      number: {{ GIT_DESCRIBE_NUMBER }}
-      string: py_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}_{{ build_ext }}  # [not cython_enabled]
-      string: py{{ python | replace(".", "") }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}_{{ build_ext }}  # [cython_enabled]
-      noarch: python                              # [not cython_enabled]
-      skip: True                                  # [cython_enabled and py<38]
-      script: >
-        python -m pip install . -vv --no-deps
-        --install-option="--with-cython=profile"  # [cython_enabled]
-      track_features:             # [cython_enabled]
-        - cythonized-scheduler    # [cython_enabled]
-      entry_points:
-        - dask-scheduler = distributed.cli.dask_scheduler:main
-        - dask-ssh = distributed.cli.dask_ssh:main
-        - dask-worker = distributed.cli.dask_worker:main
-    requirements:
-      build:
-        - {{ compiler('c') }}                     # [cython_enabled]
-      host:
-        - python >=3.8
-        - pip
-        - setuptools
-        - cython                                  # [cython_enabled]
-      run:
-        - python >=3.8
-        - click >=6.6
-        - cloudpickle >=1.5.0
-        - cytoolz >=0.8.2
-        - dask-core >={{ dask_version }}
-        - jinja2
-        - msgpack-python >=0.6.0
-        - packaging >=20.0
-        - psutil >=5.0
-        - pyyaml
-        - sortedcontainers !=2.0.0,!=2.0.1
-        - tblib >=1.6.0
-        - toolz >=0.8.2
-        - tornado >=6.0.3
-        - urllib3
-        - zict >=0.1.3
-      run_constrained:
-        - distributed-impl >={{ version }} *{{ build_ext }}
-        - openssl !=1.1.1e
-    test:
-      imports:
-        - distributed
-        - distributed.cli
-        - distributed.comm
-        - distributed.deploy
-        - distributed.diagnostics
-        - distributed.protocol
-      commands:
-        - pip check
-        - dask-scheduler --help
-        - dask-ssh --help
-        - dask-worker --help
-        - python -c "from distributed.scheduler import COMPILED; assert COMPILED is {{ cython_enabled }}"
-      requires:
-        - pip
-        - distributed-impl >={{ version }} *{{ build_ext }}
-    about:
-      home: https://distributed.dask.org
-      summary: Distributed scheduler for Dask
-      license: BSD-3-Clause
-      license_family: BSD
-      license_file: LICENSE.txt
-      description: |
-        Distributed is a lightweight library for distributed computing in Python.
-        It extends both the concurrent.futures and dask APIs to moderate sized
-        clusters.
-      doc_url: https://distributed.dask.org
-      doc_source_url: https://github.com/dask/distributed/blob/main/docs/source/index.rst
-      dev_url: https://github.com/dask/distributed
+test:
+  imports:
+    - distributed
+    - distributed.cli
+    - distributed.comm
+    - distributed.deploy
+    - distributed.diagnostics
+    - distributed.protocol
+  commands:
+    - pip check
+    - dask-scheduler --help
+    - dask-ssh --help
+    - dask-worker --help
+  requires:
+    - pip
+
+about:
+  home: https://distributed.dask.org
+  summary: Distributed scheduler for Dask
+  license: BSD-3-Clause
+  license_family: BSD
+  license_file: LICENSE.txt
+  description: |
+    Distributed is a lightweight library for distributed computing in Python.
+    It extends both the concurrent.futures and dask APIs to moderate sized
+    clusters.
+  doc_url: https://distributed.dask.org
+  doc_source_url: https://github.com/dask/distributed/blob/main/docs/source/index.rst
+  dev_url: https://github.com/dask/distributed

distributed/cli/tests/test_dask_spec.py

@@ -1,14 +1,11 @@
 import sys
 
-import pytest
 import yaml
 
 from distributed import Client
-from distributed.scheduler import COMPILED
 from distributed.utils_test import gen_cluster, gen_test, popen
 
 
-@pytest.mark.skipif(COMPILED, reason="Fails with cythonized scheduler")
 @gen_test(timeout=120)
 async def test_text():
     with popen(
@@ -38,7 +35,6 @@ async def test_text():
                 assert w["nthreads"] == 3
 
 
-@pytest.mark.skipif(COMPILED, reason="Fails with cythonized scheduler")
 @gen_cluster(client=True, nthreads=[])
 async def test_file(c, s, tmp_path):
     fn = str(tmp_path / "foo.yaml")

distributed/deploy/tests/test_local.py

@@ -18,7 +18,6 @@
 from distributed.deploy.local import LocalCluster
 from distributed.deploy.utils_test import ClusterTest
 from distributed.metrics import time
-from distributed.scheduler import COMPILED
 from distributed.system import MEMORY_LIMIT
 from distributed.utils import TimeoutError, sync
 from distributed.utils_test import (
@@ -762,7 +761,6 @@ def scale_down(self, *args, **kwargs):
     await cluster.close()
 
 
-@pytest.mark.skipif(COMPILED, reason="Fails with cythonized scheduler")
 def test_local_tls_restart(loop):
     from distributed.utils_test import tls_only_security
 

distributed/diagnostics/tests/test_progress.py

@@ -12,7 +12,6 @@
     Progress,
     SchedulerPlugin,
 )
-from distributed.scheduler import COMPILED
 from distributed.utils_test import dec, div, gen_cluster, inc, nodebug, slowdec, slowinc
 
 
@@ -93,8 +92,7 @@ def check_bar_completed(capsys, width=40):
     assert percent == "100% Completed"
 
 
-@pytest.mark.flaky(condition=not COMPILED and LINUX, reruns=10, reruns_delay=5)
-@pytest.mark.skipif(COMPILED, reason="Fails with cythonized scheduler")
+@pytest.mark.flaky(condition=LINUX, reruns=10, reruns_delay=5)
 @gen_cluster(client=True, Worker=Nanny)
 async def test_AllProgress(c, s, a, b):
     x, y, z = c.map(inc, [1, 2, 3])