fix audit and test reqs (#1160)

* fix audit and test reqs

* revert sklearn version and ignore vuln

.github/workflows/main.yml

@@ -132,7 +132,7 @@ jobs:
       - name: Install package
         run: pip install -e .[dev,spark,fsspec]
       - name: Run pip-audit
-        run: pip-audit --ignore-vuln PYSEC-2024-48
+        run: pip-audit --ignore-vuln PYSEC-2024-48 --ignore-vuln GHSA-jw8x-6495-233v
       - name: Run Tests
         run: python -m pytest --durations=50
   test:

requirements.min.txt

@@ -21,7 +21,7 @@ pyarrow==14.0.1
 pyspark==3.4.0
 fsspec[full]==2024.2.0
 certifi==2023.07.22
-urllib3==1.26.18
+urllib3==1.26.19
 ujson==5.4.0
 
 openai==1.16.2

setup.py

@@ -5,6 +5,7 @@
 # Distributed under the terms of the Modified BSD License.
 import os
 from os.path import join as pjoin
+from pathlib import Path
 
 from setuptools import setup
 
@@ -47,7 +48,7 @@
 setup_args = dict(
     cmdclass=cmdclass,
     author_email="[email protected]",
-    long_description=open("README.md").read(),
+    long_description=(Path(__file__).parent / "README.md").read_text("utf8"),
     long_description_content_type="text/markdown",
     include_package_data=True,
     install_requires=[
@@ -64,13 +65,13 @@
         "litestar>=2.8.3",
         "typing-inspect>=0.9.0",
         "uvicorn[standard]>=0.22.0",
-        "watchdog>=3",
+        "watchdog>=3.0.0",
         "typer>=0.3",
         "rich>=13",
         "iterative-telemetry>=0.0.5",
         "dynaconf>=3.2.4",
         "certifi>=2023.07.22",
-        "urllib3>=1.26.18",
+        "urllib3>=1.26.19",
         "fsspec>=2024.2.0",
         "ujson>=5.4.0",
     ],

tests/test_setup.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+
+from setup import setup_args
+
+
+def test_minimal_requirements():
+    path = Path(__file__).parent.parent
+    with open(path / "requirements.min.txt") as f:
+        lines = {line.strip().split("#")[0] for line in f.readlines()}
+        min_reqs = {k.split("[")[0]: v for line in lines if line.strip() for k, v in (line.strip().split("=="),)}
+
+    install_reqs = {k.split("[")[0]: v for r in setup_args["install_requires"] for k, v in (r.split(">="),)}
+    extra = []
+    wrong_version = []
+    for m, v in install_reqs.items():
+        if m not in min_reqs:
+            extra.append(f"{m}>={v}")
+            continue
+        if v != min_reqs[m]:
+            wrong_version.append(f"{m}>={v}")
+            continue
+
+    assert (
+        len(extra) == 0 and len(wrong_version) == 0
+    ), f"install_requires has extra reqs {extra} and wrong versions of {wrong_version}"