Add conversion methods to Request (#5)

.flake8

@@ -1,11 +1,16 @@
 [flake8]
 extend-select = TC, TC1
 ignore =
+    # D205: 1 blank line required between summary line and description
+    # D400: First line should end with a period
+    # We need longer summary lines, specially since we use Sphinx syntax.
+    D205, D400
 max-line-length = 88
 per-file-ignores =
     # F401: Imported but unused
     form2request/__init__.py:F401
     # D100-D104: Missing docstring
     docs/conf.py:D100
     tests/__init__.py:D104
+    tests/test_conversion.py:D100,D103
     tests/test_main.py:D100,D103

.github/workflows/test.yml

@@ -13,11 +13,15 @@ jobs:
         include:
         - python-version: '3.8'
           toxenv: min
+        - python-version: '3.8'
+          toxenv: min-extra
         - python-version: '3.8'
         - python-version: '3.9'
         - python-version: '3.10'
         - python-version: '3.11'
         - python-version: '3.12'
+        - python-version: '3.12'
+          toxenv: extra
 
     steps:
     - uses: actions/checkout@v4

docs/conf.py

@@ -13,7 +13,9 @@
 intersphinx_mapping = {
     "lxml": ("https://lxml.de/apidoc/", None),
     "parsel": ("https://parsel.readthedocs.io/en/stable", None),
+    "poet": ("https://web-poet.readthedocs.io/en/latest/", None),
     "python": ("https://docs.python.org/3", None),
+    "requests": ("https://requests.readthedocs.io/en/latest/", None),
     "scrapy": ("https://docs.scrapy.org/en/latest", None),
 }
 

docs/usage.rst

@@ -15,20 +15,20 @@ You can use :func:`~form2request.form2request` to generate form submission
 request data:
 
 >>> from form2request import form2request
->>> req = form2request(form)
->>> req
+>>> request_data = form2request(form)
+>>> request_data
 Request(url='https://example.com?foo=bar', method='GET', headers=[], body=b'')
 
 :func:`~form2request.form2request` does not make requests, but you can use its
-output to build requests with any HTTP client software, e.g. with the requests_
-library:
-
-.. _requests: https://requests.readthedocs.io/en/latest/
+output to build requests with any HTTP client software. It also provides
+:ref:`conversion methods for common use cases <request>`, e.g. for the
+:doc:`requests <requests:index>` library:
 
 .. _requests-example:
 
 >>> import requests
->>> requests.request(req.method, req.url, headers=req.headers, data=req.body)  # doctest: +SKIP
+>>> request = request_data.to_requests()
+>>> requests.send(request)  # doctest: +SKIP
 <Response [200]>
 
 :func:`~form2request.form2request` supports :ref:`user-defined form data
@@ -205,18 +205,28 @@ Using request data
 The output of :func:`~form2request.form2request`,
 :class:`~form2request.Request`, is a simple request data container:
 
->>> req = form2request(form)
->>> req
+>>> request_data = form2request(form)
+>>> request_data
 Request(url='https://example.com?foo=bar', method='GET', headers=[], body=b'')
 
 While :func:`~form2request.form2request` does not make requests, you can use
 its output request data to build an actual request with any HTTP client
-software, like the requests_ library (see an example :ref:`above
-<requests-example>`) or the :doc:`Scrapy <scrapy:index>` web scraping
-framework:
+software.
+
+:class:`~form2request.Request` also provides conversion methods for common use
+cases:
+
+-   :class:`~form2request.Request.to_scrapy`, for :doc:`Scrapy 1.1.0+
+    <scrapy:index>`:
+
+    >>> request_data.to_scrapy(callback=self.parse)  # doctest: +SKIP
+    <GET https://example.com?foo=bar>
+
+-   :class:`~form2request.Request.to_requests`, for :doc:`requests 1.0.0+
+    <requests:index>` (see an example :ref:`above <requests-example>`).
 
-.. _Scrapy: https://docs.scrapy.org/en/latest/
+-   :class:`~form2request.Request.to_poet`, for :doc:`web-poet 0.2.0+
+    <poet:index>`:
 
->>> from scrapy import Request
->>> Request(req.url, method=req.method, headers=req.headers, body=req.body)
-<GET https://example.com?foo=bar>
+    >>> request_data.to_poet()
+    HttpRequest(url=RequestUrl('https://example.com?foo=bar'), method='GET', headers=<HttpRequestHeaders()>, body=b'')

form2request/_base.py

@@ -178,6 +178,57 @@ class Request:
     headers: list[tuple[str, str]]
     body: bytes
 
+    def to_poet(self, **kwargs):
+        """Convert the request to :class:`web_poet.HttpRequest
+        <web_poet.page_inputs.http.HttpRequest>`.
+
+        All *kwargs* are passed to :class:`web_poet.HttpRequest
+        <web_poet.page_inputs.http.HttpRequest>` as is.
+        """
+        import web_poet
+
+        return web_poet.HttpRequest(
+            url=self.url,
+            method=self.method,
+            headers=self.headers,
+            body=self.body,
+            **kwargs,
+        )
+
+    def to_requests(self, **kwargs):
+        """Convert the request to :class:`requests.PreparedRequest`.
+
+        All *kwargs* are passed to :class:`requests.Request` as is.
+        """
+        import requests
+
+        request = requests.Request(
+            self.method,
+            self.url,
+            headers=dict(self.headers),
+            data=self.body,
+            **kwargs,
+        )
+        return request.prepare()
+
+    def to_scrapy(self, callback, **kwargs):
+        """Convert the request to :class:`scrapy.Request
+        <scrapy.http.Request>`.
+
+        All *kwargs* are passed to :class:`scrapy.Request
+        <scrapy.http.Request>` as is.
+        """
+        import scrapy  # type: ignore[import-untyped]
+
+        return scrapy.Request(
+            self.url,
+            callback=callback,
+            method=self.method,
+            headers=self.headers,
+            body=self.body,
+            **kwargs,
+        )
+
 
 def form2request(
     form: FormElement | Selector | SelectorList,

tests/test_conversion.py

@@ -0,0 +1,156 @@
+import pytest
+
+from form2request import Request
+
+web_poet = pytest.importorskip("web_poet")
+scrapy = pytest.importorskip("scrapy")
+requests = pytest.importorskip("requests")
+
+
+def fake_scrapy_callback(self, response):
+    pass
+
+
+@pytest.mark.parametrize(
+    ("request_data", "method", "kwargs", "expected"),
+    (
+        # GET
+        *(
+            (
+                Request(
+                    url="https://example.com?foo=bar",
+                    method="GET",
+                    headers=[],
+                    body=b"",
+                ),
+                method,
+                kwargs,
+                expected,
+            )
+            for method, kwargs, expected in (
+                (
+                    "poet",
+                    {},
+                    web_poet.HttpRequest(
+                        url=web_poet.RequestUrl("https://example.com?foo=bar"),
+                        method="GET",
+                        headers=web_poet.HttpRequestHeaders(),
+                        body=web_poet.HttpRequestBody(b""),
+                    ),
+                ),
+                (
+                    "requests",
+                    {},
+                    requests.Request("GET", "https://example.com?foo=bar").prepare(),
+                ),
+                (
+                    "scrapy",
+                    {"callback": fake_scrapy_callback},
+                    scrapy.Request(
+                        "https://example.com?foo=bar", callback=fake_scrapy_callback
+                    ),
+                ),
+            )
+        ),
+        # POST
+        *(
+            (
+                Request(
+                    url="https://example.com",
+                    method="POST",
+                    headers=[("Content-Type", "application/x-www-form-urlencoded")],
+                    body=b"foo=bar",
+                ),
+                method,
+                kwargs,
+                expected,
+            )
+            for method, kwargs, expected in (
+                (
+                    "poet",
+                    {},
+                    web_poet.HttpRequest(
+                        url=web_poet.RequestUrl("https://example.com"),
+                        method="POST",
+                        headers=web_poet.HttpRequestHeaders(
+                            {"Content-Type": "application/x-www-form-urlencoded"}
+                        ),
+                        body=web_poet.HttpRequestBody(b"foo=bar"),
+                    ),
+                ),
+                (
+                    "requests",
+                    {},
+                    requests.Request(
+                        "POST",
+                        "https://example.com",
+                        headers={"Content-Type": "application/x-www-form-urlencoded"},
+                        data=b"foo=bar",
+                    ).prepare(),
+                ),
+                (
+                    "scrapy",
+                    {"callback": fake_scrapy_callback},
+                    scrapy.Request(
+                        "https://example.com",
+                        method="POST",
+                        headers={"Content-Type": "application/x-www-form-urlencoded"},
+                        body=b"foo=bar",
+                        callback=fake_scrapy_callback,
+                    ),
+                ),
+            )
+        ),
+        # kwargs
+        (
+            Request(
+                url="https://example.com",
+                method="POST",
+                headers=[("Content-Type", "application/x-www-form-urlencoded")],
+                body=b"foo=bar",
+            ),
+            "requests",
+            {"params": {"foo": "bar"}},
+            requests.Request(
+                "POST",
+                "https://example.com?foo=bar",
+                headers={"Content-Type": "application/x-www-form-urlencoded"},
+                data=b"foo=bar",
+            ).prepare(),
+        ),
+        (
+            Request(
+                url="https://example.com",
+                method="POST",
+                headers=[("Content-Type", "application/x-www-form-urlencoded")],
+                body=b"foo=bar",
+            ),
+            "scrapy",
+            {"callback": fake_scrapy_callback, "meta": {"foo": "bar"}},
+            scrapy.Request(
+                "https://example.com",
+                method="POST",
+                headers={"Content-Type": "application/x-www-form-urlencoded"},
+                body=b"foo=bar",
+                callback=fake_scrapy_callback,
+                meta={"foo": "bar"},
+            ),
+        ),
+    ),
+)
+def test_conversion(request_data, method, kwargs, expected):
+    actual = getattr(request_data, f"to_{method}")(**kwargs)
+    if method == "poet":
+        for field in ("method", "headers", "body"):
+            assert getattr(actual, field) == getattr(expected, field)
+        # RequestUrl(…) != RequestUrl(…)
+        assert str(actual.url) == str(expected.url)
+    elif method == "requests":
+        # Request(…).prepare() != Request(…).prepare()
+        for field in ("url", "method", "headers", "body"):
+            assert getattr(actual, field) == getattr(expected, field)
+    else:
+        assert method == "scrapy"
+        # Request(…) != Request(…)
+        for field in ("url", "method", "headers", "body", "callback", "meta"):
+            assert getattr(actual, field) == getattr(expected, field)

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = pre-commit,mypy,docs,doctest,twinecheck,min,py38,py39,py310,py311,py312
+envlist = pre-commit,mypy,docs,doctest,twinecheck,min,min-extra,py38,py39,py310,py311,py312,extra
 
 [testenv]
 deps =
@@ -12,14 +12,35 @@ commands =
         --cov=form2request \
         {posargs:tests}
 
-[testenv:min]
-basepython = python3.8
+[min]
 deps =
     {[testenv]deps}
     lxml==4.4.1
     parsel==1.8.1
+
+[testenv:min]
+basepython = python3.8
+deps =
+    {[min]deps}
     w3lib==1.19.0
 
+[testenv:extra]
+deps =
+    {[testenv]deps}
+    requests
+    scrapy
+    web-poet
+
+[testenv:min-extra]
+basepython = {[testenv:min]basepython}
+deps =
+    {[min]deps}
+    # web-poet >= 0.2.0 requires w3lib >= 1.22.0
+    w3lib==1.22.0
+    requests==1.0.0
+    scrapy==1.1.0
+    web-poet==0.2.0
+
 [testenv:pre-commit]
 deps =
     pre-commit
@@ -29,8 +50,11 @@ commands = pre-commit run --all-files --show-diff-on-failure
 basepython = python3.12
 deps =
     mypy==1.10.0
-    pytest
     lxml-stubs
+    pytest
+    scrapy
+    types-requests
+    web-poet
 
 commands = mypy form2request tests
 
@@ -46,10 +70,8 @@ commands =
 
 [testenv:doctest]
 deps =
-    {[testenv]deps}
+    {[testenv:extra]deps}
     parsel
-    requests
-    scrapy
 commands =
     pytest \
         --doctest-glob="*.rst" --doctest-modules \