Migrate from Flake8 and Autopep8 to Ruff

CHANGELOG.md

@@ -2,7 +2,9 @@
 
 ## [1.0.5](../../releases/tag/v1.0.5) - Unreleased
 
-...
+### Internal changes
+
+- Migrate from Autopep8 and Flake8 to Ruff
 
 ## [1.0.4](../../releases/tag/v1.0.4) - 2023-10-18
 

CONTRIBUTING.md

@@ -24,11 +24,11 @@ To install this package and its development dependencies, run `make install-dev`
 
 ## Formatting
 
-We use `autopep8` and `isort` to automatically format the code to a common format. To run the formatting, just run `make format`.
+We use `ruff` and `isort` to automatically format the code to a common format. To run the formatting, just run `make format`.
 
 ## Linting, type-checking and unit testing
 
-We use `flake8` for linting, `mypy` for type checking and `pytest` for unit testing. To run these tools, just run `make check-code`.
+We use `ruff` for linting, `mypy` for type checking and `pytest` for unit testing. To run these tools, just run `make check-code`.
 
 ## Release process
 

Makefile

@@ -1,7 +1,9 @@
 .PHONY: clean install-dev build publish twine-check lint unit-tests type-check check-code format check-version-availability check-changelog-entry
 
+DIRS_WITH_CODE = src tests scripts
+
 clean:
-	rm -rf build dist .mypy_cache .pytest_cache src/*.egg-info __pycache__
+	rm -rf build dist .mypy_cache .pytest_cache .ruff_cache src/*.egg-info __pycache__
 
 install-dev:
 	python -m pip install --upgrade pip
@@ -18,22 +20,22 @@ twine-check:
 	python -m twine check dist/*
 
 lint:
-	python3 -m flake8
+	python -m ruff check $(DIRS_WITH_CODE)
 
 unit-tests:
-	python3 -m pytest -n auto -ra tests/unit
+	python -m pytest -n auto -ra tests/unit
 
 type-check:
-	python3 -m mypy
+	python -m mypy $(DIRS_WITH_CODE)
 
 check-code: lint type-check unit-tests
 
 format:
-	python3 -m isort src tests
-	python3 -m autopep8 --in-place --recursive src tests
+	python -m ruff check --fix $(DIRS_WITH_CODE)
+	python -m ruff format $(DIRS_WITH_CODE)
 
 check-version-availability:
-	python3 scripts/check_version_availability.py
+	python scripts/check_version_availability.py
 
 check-changelog-entry:
-	python3 scripts/check_version_in_changelog.py
+	python scripts/check_version_in_changelog.py

pyproject.toml

@@ -23,25 +23,9 @@ version = "1.0.5"
 
 [project.optional-dependencies]
 dev = [
-    "autopep8 ~= 2.0.4",
     "build ~= 1.0.3",
     "filelock ~= 3.12.4",
-    "flake8 ~= 6.1.0",
-    "flake8-bugbear ~= 23.9.16",
-    "flake8-commas ~= 2.1.0; python_version < '3.12'",
-    "flake8-comprehensions ~= 3.14.0",
-    "flake8-datetimez ~= 20.10.0",
-    "flake8-docstrings ~= 1.7.0",
-    "flake8-encodings ~= 0.5.0",
-    "flake8-isort ~= 6.1.0",
-    "flake8-noqa ~= 1.3.1; python_version < '3.12'",
-    "flake8-pytest-style ~= 1.7.2",
-    "flake8-quotes ~= 3.3.2; python_version < '3.12'",
-    "flake8-simplify ~= 0.21.0",
-    "flake8-unused-arguments ~= 0.0.13",
-    "isort ~= 5.12.0",
     "mypy ~= 1.5.1",
-    "pep8-naming ~= 0.13.3",
     "pre-commit ~= 3.4.0",
     "pydoc-markdown ~= 4.8.2",
     "pytest ~= 7.4.2",
@@ -50,6 +34,7 @@ dev = [
     "pytest-timeout ~= 2.2.0",
     "pytest-xdist ~= 3.3.1",
     "respx ~= 0.20.1",
+    "ruff ~= 0.1.5",
     "twine ~= 4.0.2",
 ]
 
@@ -69,3 +54,41 @@ where = ["src"]
 
 [tool.setuptools.package-data]
 apify_shared = ["py.typed"]
+
+[tool.ruff]
+line-length = 120
+select = ["ALL"]
+ignore = [
+    "BLE001",  # Do not catch blind exception
+    "COM812",  # This rule may cause conflicts when used with the formatter
+    "D100",    # Missing docstring in public module
+    "D104",    # Missing docstring in public package
+    "EM",      # flake8-errmsg
+    "ISC001",  # This rule may cause conflicts when used with the formatter
+    "FIX",     # flake8-fixme
+    "PGH003",  # Use specific rule codes when ignoring type issues
+    "PLR0913", # Too many arguments in function definition
+    "PTH123",  # `open()` should be replaced by `Path.open()`
+    "S102",    # Use of `exec` detected
+    "S105",    # Possible hardcoded password assigned to
+    "TRY003",  # Avoid specifying long messages outside the exception class
+]
+
+[tool.ruff.format]
+quote-style = "single"
+indent-style = "space"
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]
+"**/{scripts}/*" = ["D", "INP"]
+"**/{tests}/*" = ["D", "INP", "S101"]
+
+[tool.ruff.lint.flake8-quotes]
+docstring-quotes = "double"
+inline-quotes = "single"
+
+[tool.ruff.lint.isort]
+known-first-party = ["apify", "apify_client", "apify_shared"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"

scripts/check_version_availability.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+
 from utils import get_current_package_version, get_published_package_versions
 
 # Checks whether the current package version number was not already used in a published release.

scripts/check_version_in_changelog.py

@@ -15,8 +15,11 @@
 
     with open(CHANGELOG_PATH, encoding='utf-8') as changelog_file:
         for line in changelog_file:
-            # The heading for the changelog entry for the given version can start with either the version number, or the version number in a link
-            if re.match(fr'(## )\[?{current_package_version}([\] ]|$)', line):
+            # The heading for the changelog entry for the given version can start with either the version number,
+            # or the version number in a link
+            if re.match(rf'(## )\[?{current_package_version}([\] ]|$)', line):
                 break
         else:
-            raise RuntimeError(f'There is no entry in the changelog for the current package version ({current_package_version})')
+            raise RuntimeError(
+                f'There is no entry in the changelog for the current package version ({current_package_version})'
+            )

scripts/print_current_package_version.py

@@ -4,4 +4,4 @@
 
 # Print the current package version from the pyproject.toml file to stdout
 if __name__ == '__main__':
-    print(get_current_package_version(), end='')
+    print(get_current_package_version(), end='')  # noqa: T201

scripts/update_version_for_prerelease.py

@@ -9,7 +9,7 @@
 # and if not, modifies the package version number in pyproject.toml
 # from a stable release version (X.Y.Z) to a prerelease version (X.Y.ZbN or X.Y.Z.aN or X.Y.Z.rcN)
 if __name__ == '__main__':
-    if len(sys.argv) != 2:
+    if len(sys.argv) != 2:  # noqa: PLR2004
         raise RuntimeError('You must pass the prerelease type as an argument to this script!')
 
     prerelease_type = sys.argv[1]
@@ -27,7 +27,9 @@
 
     # We can only transform a stable release version (X.Y.Z) to a prerelease version (X.Y.ZxxxN)
     if not re.match(r'^\d+\.\d+\.\d+$', current_version):
-        raise RuntimeError(f'The current version {current_version} does not match the proper semver format for stable releases (X.Y.Z)')
+        raise RuntimeError(
+            f'The current version {current_version} does not match the proper semver format for stable releases (X.Y.Z)'
+        )
 
     # Load the version numbers of the currently published versions from PyPI
     published_versions = get_published_package_versions()

scripts/utils.py

@@ -1,6 +1,7 @@
 import json
 import pathlib
-import urllib.request
+from urllib.error import HTTPError
+from urllib.request import urlopen
 
 PACKAGE_NAME = 'apify_shared'
 REPO_ROOT = pathlib.Path(__file__).parent.resolve() / '..'
@@ -10,13 +11,12 @@
 # Load the current version number from pyproject.toml
 # It is on a line in the format `version = "1.2.3"`
 def get_current_package_version() -> str:
-    with open(PYPROJECT_TOML_FILE_PATH, 'r', encoding='utf-8') as pyproject_toml_file:
+    with open(PYPROJECT_TOML_FILE_PATH, encoding='utf-8') as pyproject_toml_file:
         for line in pyproject_toml_file:
             if line.startswith('version = '):
                 delim = '"' if '"' in line else "'"
-                version = line.split(delim)[1]
-                return version
-        else:
+                return line.split(delim)[1]
+        else:  # noqa: PLW0120
             raise RuntimeError('Unable to find version string.')
 
 
@@ -29,7 +29,7 @@ def set_current_package_version(version: str) -> None:
         for line in pyproject_toml_file:
             if line.startswith('version = '):
                 version_string_found = True
-                line = f'version = "{version}"\n'
+                line = f'version = "{version}"\n'  # noqa: PLW2901
             updated_pyproject_toml_file_lines.append(line)
 
         if not version_string_found:
@@ -44,11 +44,11 @@ def set_current_package_version(version: str) -> None:
 def get_published_package_versions() -> list:
     package_info_url = f'https://pypi.org/pypi/{PACKAGE_NAME}/json'
     try:
-        package_data = json.load(urllib.request.urlopen(package_info_url))
+        package_data = json.load(urlopen(package_info_url))  # noqa: S310
         published_versions = list(package_data['releases'].keys())
     # If the URL returns 404, it means the package has no releases yet (which is okay in our case)
-    except urllib.error.HTTPError as e:
-        if e.code != 404:
-            raise e
+    except HTTPError as e:
+        if e.code != 404:  # noqa: PLR2004
+            raise
         published_versions = []
     return published_versions

src/apify_shared/consts.py

@@ -1,5 +1,7 @@
+from __future__ import annotations
+
 from enum import Enum
-from typing import List, Literal, get_args
+from typing import Literal, get_args
 
 
 class ActorJobStatus(str, Enum):
@@ -23,9 +25,14 @@ class ActorJobStatus(str, Enum):
     ABORTED = 'ABORTED'
 
     @property
-    def _is_terminal(self) -> bool:
+    def _is_terminal(self: ActorJobStatus) -> bool:
         """Whether this actor job status is terminal."""
-        return self in (ActorJobStatus.SUCCEEDED, ActorJobStatus.FAILED, ActorJobStatus.TIMED_OUT, ActorJobStatus.ABORTED)
+        return self in (
+            ActorJobStatus.SUCCEEDED,
+            ActorJobStatus.FAILED,
+            ActorJobStatus.TIMED_OUT,
+            ActorJobStatus.ABORTED,
+        )
 
 
 class ActorSourceType(str, Enum):
@@ -57,7 +64,7 @@ class ActorEventTypes(str, Enum):
 class ActorEnvVars(str, Enum):
     """Possible Apify-specific environment variables prefixed with "ACTOR_"."""
 
-    # TODO: document these
+    # TODO: document these  # noqa: TD002, TD003
 
     #: BUILD_ID
     BUILD_ID = 'ACTOR_BUILD_ID'
@@ -96,7 +103,7 @@ class ActorEnvVars(str, Enum):
 class ApifyEnvVars(str, Enum):
     """Possible Apify-specific environment variables prefixed with "APIFY_"."""
 
-    # TODO: document these
+    # TODO: document these  # noqa: TD002, TD003
 
     #: API_BASE_URL
     API_BASE_URL = 'APIFY_API_BASE_URL'
@@ -273,13 +280,11 @@ class MetaOrigin(str, Enum):
     ApifyEnvVars.SYSTEM_INFO_INTERVAL_MILLIS,
 ]
 
-INTEGER_ENV_VARS: List[INTEGER_ENV_VARS_TYPE] = list(get_args(INTEGER_ENV_VARS_TYPE))
+INTEGER_ENV_VARS: list[INTEGER_ENV_VARS_TYPE] = list(get_args(INTEGER_ENV_VARS_TYPE))
 
-FLOAT_ENV_VARS_TYPE = Literal[
-    ApifyEnvVars.MAX_USED_CPU_RATIO,
-]
+FLOAT_ENV_VARS_TYPE = Literal[ApifyEnvVars.MAX_USED_CPU_RATIO,]
 
-FLOAT_ENV_VARS: List[FLOAT_ENV_VARS_TYPE] = list(get_args(FLOAT_ENV_VARS_TYPE))
+FLOAT_ENV_VARS: list[FLOAT_ENV_VARS_TYPE] = list(get_args(FLOAT_ENV_VARS_TYPE))
 
 BOOL_ENV_VARS_TYPE = Literal[
     ApifyEnvVars.DISABLE_BROWSER_SANDBOX,
@@ -291,7 +296,7 @@ class MetaOrigin(str, Enum):
     ApifyEnvVars.XVFB,
 ]
 
-BOOL_ENV_VARS: List[BOOL_ENV_VARS_TYPE] = list(get_args(BOOL_ENV_VARS_TYPE))
+BOOL_ENV_VARS: list[BOOL_ENV_VARS_TYPE] = list(get_args(BOOL_ENV_VARS_TYPE))
 
 DATETIME_ENV_VARS_TYPE = Literal[
     # Actor env vars
@@ -302,7 +307,7 @@ class MetaOrigin(str, Enum):
     ApifyEnvVars.TIMEOUT_AT,
 ]
 
-DATETIME_ENV_VARS: List[DATETIME_ENV_VARS_TYPE] = list(get_args(DATETIME_ENV_VARS_TYPE))
+DATETIME_ENV_VARS: list[DATETIME_ENV_VARS_TYPE] = list(get_args(DATETIME_ENV_VARS_TYPE))
 
 STRING_ENV_VARS_TYPE = Literal[
     # Actor env vars
@@ -350,4 +355,4 @@ class MetaOrigin(str, Enum):
     ApifyEnvVars.WORKFLOW_KEY,
 ]
 
-STRING_ENV_VARS: List[STRING_ENV_VARS_TYPE] = list(get_args(STRING_ENV_VARS_TYPE))
+STRING_ENV_VARS: list[STRING_ENV_VARS_TYPE] = list(get_args(STRING_ENV_VARS_TYPE))

src/apify_shared/models.py

@@ -1,4 +1,6 @@
-from typing import Dict, Generic, List, TypeVar
+from __future__ import annotations
+
+from typing import Generic, TypeVar
 
 from .utils import ignore_docs
 
@@ -9,7 +11,7 @@ class ListPage(Generic[T]):
     """A single page of items returned from a list() method."""
 
     #: list: List of returned objects on this page
-    items: List[T]
+    items: list[T]
     #: int: Count of the returned objects on this page
     count: int
     #: int: The limit on the number of returned objects offset specified in the API call
@@ -22,7 +24,7 @@ class ListPage(Generic[T]):
     desc: bool
 
     @ignore_docs
-    def __init__(self, data: Dict) -> None:
+    def __init__(self: ListPage, data: dict) -> None:
         """Initialize a ListPage instance from the API response data."""
         self.items = data['items'] if 'items' in data else []
         self.offset = data['offset'] if 'offset' in data else 0

src/apify_shared/types.py

@@ -1,6 +1,8 @@
 from typing import Any, Dict, List, Union
 
-# Type for representing json-serializable values
-# It's close enough to the real thing supported by json.parse, and the best we can do until mypy supports recursive types
-# It was suggested in a discussion with (and approved by) Guido van Rossum, so I'd consider it correct enough
+# Type for representing json-serializable values.
+# It's close enough to the real thing supported by json.parse,
+# and the best we can do until mypy supports recursive types.
+# It was suggested in a discussion with (and approved by) Guido van Rossum,
+# so I'd consider it correct enough.
 JSONSerializable = Union[str, int, float, bool, None, Dict[str, Any], List[Any]]