diff --git a/curl_cffi/__init__.py b/curl_cffi/__init__.py index 263fe113..fe611555 100644 --- a/curl_cffi/__init__.py +++ b/curl_cffi/__init__.py @@ -1,5 +1,7 @@ __all__ = [ "Curl", + "CurlChrome", + "CurlFirefox", "CurlInfo", "CurlOpt", "CurlMOpt", @@ -7,16 +9,23 @@ "CurlHttpVersion", "CurlError", "AsyncCurl", + "AsyncCurlChrome", + "AsyncCurlFirefox", "ffi", "lib", + "ffi_ff", + "lib_ff", ] + import _cffi_backend # noqa: F401 # required by _wrapper -# This line includes _wrapper.so into the wheel -from ._wrapper import ffi, lib # type: ignore +# This line includes _wrapper_{chrome,ff}.so into the wheel +from ._wrapper_chrome import ffi, lib +from ._wrapper_ff import ffi as ffi_ff, lib as lib_ff + from .const import CurlInfo, CurlMOpt, CurlOpt, CurlECode, CurlHttpVersion -from .curl import Curl, CurlError -from .aio import AsyncCurl +from .curl import Curl, CurlChrome, CurlFirefox, CurlError +from .aio import AsyncCurl, AsyncCurlChrome, AsyncCurlFirefox -from .__version__ import __title__, __version__, __description__, __curl_version__ +from .__version__ import __title__, __version__, __description__, __curl_version__, __curl_chrome_version__, __curl_firefox_version__ diff --git a/curl_cffi/__version__.py b/curl_cffi/__version__.py index b0cfd8d3..43a2ea15 100644 --- a/curl_cffi/__version__.py +++ b/curl_cffi/__version__.py @@ -1,6 +1,6 @@ # New in version 3.8. # from importlib import metadata -from .curl import Curl +from .curl import Curl, CurlFirefox __title__ = "curl_cffi" @@ -9,3 +9,5 @@ __description__ = "libcurl ffi bindings for Python, with impersonation support" __version__ = "0.5.10" __curl_version__ = Curl().version().decode() +__curl_chrome_version__ = __curl_version__ +__curl_firefox_version__ = CurlFirefox().version().decode() diff --git a/curl_cffi/aio.py b/curl_cffi/aio.py index 3bc2ffc7..80d296f2 100644 --- a/curl_cffi/aio.py +++ b/curl_cffi/aio.py @@ -4,7 +4,8 @@ from typing import Any from weakref import WeakSet, WeakKeyDictionary -from ._wrapper import ffi, lib # type: ignore +from ._wrapper_chrome import ffi as ffi_chrome, lib as lib_chrome # type: ignore +from ._wrapper_ff import ffi as ffi_ff, lib as lib_ff # type: ignore from .const import CurlMOpt from .curl import Curl, DEFAULT_CACERT @@ -75,54 +76,60 @@ def _get_selector(loop) -> asyncio.AbstractEventLoop: CURLMSG_DONE = 1 -@ffi.def_extern() -def timer_function(curlm, timeout_ms: int, clientp: Any): - """ - see: https://curl.se/libcurl/c/CURLMOPT_TIMERFUNCTION.html - """ - async_curl = ffi.from_handle(clientp) - # print("time out in %sms" % timeout_ms) - if timeout_ms == -1: - for timer in async_curl._timers: - timer.cancel() - async_curl._timers = WeakSet() - else: - timer = async_curl.loop.call_later( - timeout_ms / 1000, - async_curl.process_data, - CURL_SOCKET_TIMEOUT, # -1 - CURL_POLL_NONE, # 0 - ) - async_curl._timers.add(timer) - - -@ffi.def_extern() -def socket_function(curl, sockfd: int, what: int, clientp: Any, data: Any): - async_curl = ffi.from_handle(clientp) - loop = async_curl.loop - - if what & CURL_POLL_IN or what & CURL_POLL_OUT or what & CURL_POLL_REMOVE: - if sockfd in async_curl._sockfds: - loop.remove_reader(sockfd) - loop.remove_writer(sockfd) - async_curl._sockfds.remove(sockfd) - elif what & CURL_POLL_REMOVE: - message = f"File descriptor {sockfd} not found." - raise TypeError(message) - - if what & CURL_POLL_IN: - loop.add_reader(sockfd, async_curl.process_data, sockfd, CURL_CSELECT_IN) - async_curl._sockfds.add(sockfd) - if what & CURL_POLL_OUT: - loop.add_writer(sockfd, async_curl.process_data, sockfd, CURL_CSELECT_OUT) - async_curl._sockfds.add(sockfd) +for linked in [(ffi_chrome, lib_chrome), (ffi_ff, lib_ff)]: + ffi, lib = linked + + @ffi.def_extern() + def timer_function(curlm, timeout_ms: int, clientp: Any): + """ + see: https://curl.se/libcurl/c/CURLMOPT_TIMERFUNCTION.html + """ + async_curl = ffi.from_handle(clientp) + # print("time out in %sms" % timeout_ms) + if timeout_ms == -1: + for timer in async_curl._timers: + timer.cancel() + async_curl._timers = WeakSet() + else: + timer = async_curl.loop.call_later( + timeout_ms / 1000, + async_curl.process_data, + CURL_SOCKET_TIMEOUT, # -1 + CURL_POLL_NONE, # 0 + ) + async_curl._timers.add(timer) + + + @ffi.def_extern() + def socket_function(curl, sockfd: int, what: int, clientp: Any, data: Any): + async_curl = ffi.from_handle(clientp) + loop = async_curl.loop + + if what & CURL_POLL_IN or what & CURL_POLL_OUT or what & CURL_POLL_REMOVE: + if sockfd in async_curl._sockfds: + loop.remove_reader(sockfd) + loop.remove_writer(sockfd) + async_curl._sockfds.remove(sockfd) + elif what & CURL_POLL_REMOVE: + message = f"File descriptor {sockfd} not found." + raise TypeError(message) + + if what & CURL_POLL_IN: + loop.add_reader(sockfd, async_curl.process_data, sockfd, CURL_CSELECT_IN) + async_curl._sockfds.add(sockfd) + if what & CURL_POLL_OUT: + loop.add_writer(sockfd, async_curl.process_data, sockfd, CURL_CSELECT_OUT) + async_curl._sockfds.add(sockfd) + class AsyncCurl: """Wrapper around curl_multi handle to provide asyncio support. It uses the libcurl socket_action APIs.""" - def __init__(self, cacert: str = DEFAULT_CACERT, loop=None): - self._curlm = lib.curl_multi_init() + def __init__(self, cacert: str = DEFAULT_CACERT, loop = None, _ffi = ffi_chrome, _lib = lib_chrome): + self._ffi = _ffi + self._lib = _lib + self._curlm = self._lib.curl_multi_init() self._cacert = cacert self._curl2future = {} # curl to future map self._curl2curl = {} # c curl to Curl @@ -135,9 +142,9 @@ def __init__(self, cacert: str = DEFAULT_CACERT, loop=None): self._setup() def _setup(self): - self.setopt(CurlMOpt.TIMERFUNCTION, lib.timer_function) - self.setopt(CurlMOpt.SOCKETFUNCTION, lib.socket_function) - self._self_handle = ffi.new_handle(self) + self.setopt(CurlMOpt.TIMERFUNCTION, self._lib.timer_function) + self.setopt(CurlMOpt.SOCKETFUNCTION, self._lib.socket_function) + self._self_handle = self._ffi.new_handle(self) self.setopt(CurlMOpt.SOCKETDATA, self._self_handle) self.setopt(CurlMOpt.TIMERDATA, self._self_handle) @@ -147,11 +154,11 @@ def close(self): self._checker.cancel() # Close all pending futures for curl, future in self._curl2future.items(): - lib.curl_multi_remove_handle(self._curlm, curl._curl) + self._lib.curl_multi_remove_handle(self._curlm, curl._curl) if not future.done() and not future.cancelled(): future.set_result(None) # Cleanup curl_multi handle - lib.curl_multi_cleanup(self._curlm) + self._lib.curl_multi_cleanup(self._curlm) self._curlm = None # Remove add readers and writers for sockfd in self._sockfds: @@ -174,7 +181,7 @@ def add_handle(self, curl: Curl): `perform` in the async world.""" # import pdb; pdb.set_trace() curl._ensure_cacert() - lib.curl_multi_add_handle(self._curlm, curl._curl) + self._lib.curl_multi_add_handle(self._curlm, curl._curl) future = self.loop.create_future() self._curl2future[curl] = future self._curl2curl[curl._curl] = curl @@ -182,8 +189,8 @@ def add_handle(self, curl: Curl): def socket_action(self, sockfd: int, ev_bitmask: int) -> int: """Call libcurl socket_action function""" - running_handle = ffi.new("int *") - lib.curl_multi_socket_action(self._curlm, sockfd, ev_bitmask, running_handle) + running_handle = self._ffi.new("int *") + self._lib.curl_multi_socket_action(self._curlm, sockfd, ev_bitmask, running_handle) return running_handle[0] def process_data(self, sockfd: int, ev_bitmask: int): @@ -194,11 +201,11 @@ def process_data(self, sockfd: int, ev_bitmask: int): self.socket_action(sockfd, ev_bitmask) - msg_in_queue = ffi.new("int *") + msg_in_queue = self._ffi.new("int *") while True: - curl_msg = lib.curl_multi_info_read(self._curlm, msg_in_queue) + curl_msg = self._lib.curl_multi_info_read(self._curlm, msg_in_queue) # print("message in queue", msg_in_queue[0], curl_msg) - if curl_msg == ffi.NULL: + if curl_msg == self._ffi.NULL: break if curl_msg.msg == CURLMSG_DONE: # print("curl_message", curl_msg.msg, curl_msg.data.result) @@ -213,7 +220,7 @@ def process_data(self, sockfd: int, ev_bitmask: int): print("NOT DONE") # Will not reach, for no other code being defined. def _pop_future(self, curl: Curl): - lib.curl_multi_remove_handle(self._curlm, curl._curl) + self._lib.curl_multi_remove_handle(self._curlm, curl._curl) self._curl2curl.pop(curl._curl, None) return self._curl2future.pop(curl, None) @@ -237,4 +244,14 @@ def set_exception(self, curl: Curl, exception): def setopt(self, option, value): """Wrapper around curl_multi_setopt.""" - return lib.curl_multi_setopt(self._curlm, option, value) + return self._lib.curl_multi_setopt(self._curlm, option, value) + + +class AsyncCurlChrome(AsyncCurl): + pass + + +class AsyncCurlFirefox(AsyncCurl): + + def __init__(self, loop = None): + super().__init__(cacert=None, loop=loop, _ffi=ffi_ff, _lib=lib_ff) \ No newline at end of file diff --git a/curl_cffi/build.py b/curl_cffi/build.py index 67e047b5..6d76533f 100644 --- a/curl_cffi/build.py +++ b/curl_cffi/build.py @@ -3,42 +3,43 @@ from cffi import FFI -ffibuilder = FFI() # arch = "%s-%s" % (os.uname().sysname, os.uname().machine) uname = platform.uname() -ffibuilder.set_source( - "curl_cffi._wrapper", - """ - #include "shim.h" - """, - libraries=["curl-impersonate-chrome"] if uname.system != "Windows" else ["libcurl"], - library_dirs=[ - "/Users/runner/work/_temp/install/lib" - if uname.system == "Darwin" and uname.machine == "x86_64" - else "./lib" - if uname.system == "Windows" - else "/usr/local/lib" # Linux and macOS arm64 - ], - source_extension=".c", - include_dirs=[ - os.path.join(os.path.dirname(__file__), "include"), - os.path.join(os.path.dirname(__file__), "ffi"), - ], - sources=[ - os.path.join(os.path.dirname(__file__), "ffi/shim.c"), - ], - extra_compile_args=( - ["-Wno-implicit-function-declaration"] if uname.system == "Darwin" else [] - ), - # extra_link_args=["-Wl,-rpath,$ORIGIN/../libcurl/" + arch], -) +for distro in ["chrome", "ff"]: + ffibuilder = FFI() + ffibuilder.set_source( + "curl_cffi._wrapper_"+distro, + """ + #include "shim.h" + """, + libraries=["curl-impersonate-"+distro] if uname.system != "Windows" else ["libcurl"], + library_dirs=[ + "/Users/runner/work/_temp/install/lib" + if uname.system == "Darwin" and uname.machine == "x86_64" + else "./lib" + if uname.system == "Windows" + else "/usr/local/lib" # Linux and macOS arm64 + ], + source_extension=".c", + include_dirs=[ + os.path.join(os.path.dirname(__file__), "include"), + os.path.join(os.path.dirname(__file__), "ffi"), + ], + sources=[ + os.path.join(os.path.dirname(__file__), "ffi/shim.c"), + ], + extra_compile_args=( + ["-Wno-implicit-function-declaration"] if uname.system == "Darwin" else [] + ), + # extra_link_args=["-Wl,-rpath,$ORIGIN/../libcurl/" + arch], + ) -with open(os.path.join(os.path.dirname(__file__), "ffi/cdef.c")) as f: - cdef_content = f.read() - ffibuilder.cdef(cdef_content) + with open(os.path.join(os.path.dirname(__file__), "ffi/cdef.c")) as f: + cdef_content = f.read() + ffibuilder.cdef(cdef_content) -if __name__ == "__main__": - ffibuilder.compile(verbose=False) + if __name__ == "__main__": + ffibuilder.compile(verbose=False) diff --git a/curl_cffi/curl.py b/curl_cffi/curl.py index ec195823..48f37bc3 100644 --- a/curl_cffi/curl.py +++ b/curl_cffi/curl.py @@ -4,7 +4,8 @@ from http.cookies import SimpleCookie from typing import Any, List, Tuple, Union -from ._wrapper import ffi, lib # type: ignore +from ._wrapper_chrome import ffi as ffi_chrome, lib as lib_chrome # type: ignore +from ._wrapper_ff import ffi as ffi_ff, lib as lib_ff # type: ignore from .const import CurlHttpVersion, CurlInfo, CurlOpt try: @@ -34,46 +35,49 @@ def __init__(self, msg, code: int = 0, *args, **kwargs): CURL_WRITEFUNC_ERROR = 0xFFFFFFFF -@ffi.def_extern() -def debug_function(curl, type: int, data, size, clientp) -> int: - text = ffi.buffer(data, size)[:] - if type in (CURLINFO_SSL_DATA_IN, CURLINFO_SSL_DATA_OUT): - print("SSL OUT", text) - elif type in (CURLINFO_DATA_IN, CURLINFO_DATA_OUT): - print(text.decode()) - else: - print(text.decode(), end="") - return 0 +for linked in [(ffi_chrome, lib_chrome), (ffi_ff, lib_ff)]: + ffi, lib = linked + @ffi.def_extern() + def debug_function(curl, type: int, data, size, clientp) -> int: + text = ffi.buffer(data, size)[:] + if type in (CURLINFO_SSL_DATA_IN, CURLINFO_SSL_DATA_OUT): + print("SSL OUT", text) + elif type in (CURLINFO_DATA_IN, CURLINFO_DATA_OUT): + print(text.decode()) + else: + print(text.decode(), end="") + return 0 -@ffi.def_extern() -def buffer_callback(ptr, size, nmemb, userdata): - # assert size == 1 - buffer = ffi.from_handle(userdata) - buffer.write(ffi.buffer(ptr, nmemb)[:]) - return nmemb * size -def ensure_int(s): - if not s: - return 0 - return int(s) - -@ffi.def_extern() -def write_callback(ptr, size, nmemb, userdata): - # although similar enough to the function above, kept here for performance reasons - callback = ffi.from_handle(userdata) - wrote = callback(ffi.buffer(ptr, nmemb)[:]) - wrote = ensure_int(wrote) - if wrote == CURL_WRITEFUNC_PAUSE or wrote == CURL_WRITEFUNC_ERROR: - return wrote - # should make this an exception in future versions - if wrote != nmemb * size: - warnings.warn("Wrote bytes != received bytes.", RuntimeWarning) - return nmemb * size + @ffi.def_extern() + def buffer_callback(ptr, size, nmemb, userdata): + # assert size == 1 + buffer = ffi.from_handle(userdata) + buffer.write(ffi.buffer(ptr, nmemb)[:]) + return nmemb * size + + def ensure_int(s): + if not s: + return 0 + return int(s) + + @ffi.def_extern() + def write_callback(ptr, size, nmemb, userdata): + # although similar enough to the function above, kept here for performance reasons + callback = ffi.from_handle(userdata) + wrote = callback(ffi.buffer(ptr, nmemb)[:]) + wrote = ensure_int(wrote) + if wrote == CURL_WRITEFUNC_PAUSE or wrote == CURL_WRITEFUNC_ERROR: + return wrote + # should make this an exception in future versions + if wrote != nmemb * size: + warnings.warn("Wrote bytes != received bytes.", RuntimeWarning) + return nmemb * size # Credits: @alexio777 on https://github.com/yifeikong/curl_cffi/issues/4 -def slist_to_list(head) -> List[bytes]: +def slist_to_list(ffi, lib, head) -> List[bytes]: result = [] ptr = head while ptr: @@ -88,32 +92,34 @@ class Curl: Wrapper for `curl_easy_*` functions of libcurl. """ - def __init__(self, cacert: str = DEFAULT_CACERT, debug: bool = False, handle = None): + def __init__(self, cacert: str = DEFAULT_CACERT, debug: bool = False, handle = None, _ffi = ffi_chrome, _lib = lib_chrome): """ Parameters: cacert: CA cert path to use, by default, curl_cffi uses its own bundled cert. debug: whether to show curl debug messages. """ - self._curl = lib.curl_easy_init() if not handle else handle - self._headers = ffi.NULL - self._resolve = ffi.NULL + self._ffi = _ffi + self._lib = _lib + self._curl = self._lib.curl_easy_init() if not handle else handle + self._headers = self._ffi.NULL + self._resolve = self._ffi.NULL self._cacert = cacert self._is_cert_set = False self._write_handle = None self._header_handle = None self._body_handle = None # TODO: use CURL_ERROR_SIZE - self._error_buffer = ffi.new("char[]", 256) + self._error_buffer = self._ffi.new("char[]", 256) self._debug = debug self._set_error_buffer() def _set_error_buffer(self): - ret = lib._curl_easy_setopt(self._curl, CurlOpt.ERRORBUFFER, self._error_buffer) + ret = self._lib._curl_easy_setopt(self._curl, CurlOpt.ERRORBUFFER, self._error_buffer) if ret != 0: warnings.warn("Failed to set error buffer") if self._debug: self.setopt(CurlOpt.VERBOSE, 1) - lib._curl_easy_setopt(self._curl, CurlOpt.DEBUGFUNCTION, lib.debug_function) + self._lib._curl_easy_setopt(self._curl, CurlOpt.DEBUGFUNCTION, self._lib.debug_function) def __del__(self): self.close() @@ -125,7 +131,7 @@ def _check_error(self, errcode: int, *args): def _get_error(self, errcode: int, *args): if errcode != 0: - errmsg = ffi.string(self._error_buffer).decode() + errmsg = self._ffi.string(self._error_buffer).decode() action = " ".join([str(a) for a in args]) return CurlError( f"Failed to {action}, ErrCode: {errcode}, Reason: '{errmsg}'. " @@ -154,28 +160,28 @@ def setopt(self, option: CurlOpt, value: Any): # Convert value value_type = input_option.get(int(option / 10000) * 10000) if value_type == "int*": - c_value = ffi.new("int*", value) + c_value = self._ffi.new("int*", value) elif option == CurlOpt.WRITEDATA: - c_value = ffi.new_handle(value) + c_value = self._ffi.new_handle(value) self._write_handle = c_value - lib._curl_easy_setopt( - self._curl, CurlOpt.WRITEFUNCTION, lib.buffer_callback + self._lib._curl_easy_setopt( + self._curl, CurlOpt.WRITEFUNCTION, self._lib.buffer_callback ) elif option == CurlOpt.HEADERDATA: - c_value = ffi.new_handle(value) + c_value = self._ffi.new_handle(value) self._header_handle = c_value - lib._curl_easy_setopt( - self._curl, CurlOpt.HEADERFUNCTION, lib.buffer_callback + self._lib._curl_easy_setopt( + self._curl, CurlOpt.HEADERFUNCTION, self._lib.buffer_callback ) elif option == CurlOpt.WRITEFUNCTION: - c_value = ffi.new_handle(value) + c_value = self._ffi.new_handle(value) self._write_handle = c_value - lib._curl_easy_setopt(self._curl, CurlOpt.WRITEFUNCTION, lib.write_callback) + self._lib._curl_easy_setopt(self._curl, CurlOpt.WRITEFUNCTION, self._lib.write_callback) option = CurlOpt.WRITEDATA elif option == CurlOpt.HEADERFUNCTION: - c_value = ffi.new_handle(value) + c_value = self._ffi.new_handle(value) self._header_handle = c_value - lib._curl_easy_setopt(self._curl, CurlOpt.WRITEFUNCTION, lib.write_callback) + self._lib._curl_easy_setopt(self._curl, CurlOpt.WRITEFUNCTION, self._lib.write_callback) option = CurlOpt.HEADERDATA elif value_type == "char*": if isinstance(value, str): @@ -190,16 +196,16 @@ def setopt(self, option: CurlOpt, value: Any): if option == CurlOpt.HTTPHEADER: for header in value: - self._headers = lib.curl_slist_append(self._headers, header) - ret = lib._curl_easy_setopt(self._curl, option, self._headers) + self._headers = self._lib.curl_slist_append(self._headers, header) + ret = self._lib._curl_easy_setopt(self._curl, option, self._headers) elif option == CurlOpt.RESOLVE: for resolve in value: if isinstance(resolve, str): resolve = resolve.encode() - self._resolve = lib.curl_slist_append(self._resolve, resolve) - ret = lib._curl_easy_setopt(self._curl, option, self._resolve) + self._resolve = self._lib.curl_slist_append(self._resolve, resolve) + ret = self._lib._curl_easy_setopt(self._curl, option, self._resolve) else: - ret = lib._curl_easy_setopt(self._curl, option, c_value) + ret = self._lib._curl_easy_setopt(self._curl, option, c_value) self._check_error(ret, "setopt", option, value) if option == CurlOpt.CAINFO: @@ -220,23 +226,23 @@ def getinfo(self, option: CurlInfo) -> Union[bytes, int, float, List]: 0x400000: "struct curl_slist **", } ret_cast_option = { - 0x100000: ffi.string, + 0x100000: self._ffi.string, 0x200000: int, 0x300000: float, } - c_value = ffi.new(ret_option[option & 0xF00000]) - ret = lib.curl_easy_getinfo(self._curl, option, c_value) + c_value = self._ffi.new(ret_option[option & 0xF00000]) + ret = self._lib.curl_easy_getinfo(self._curl, option, c_value) self._check_error(ret, "getinfo", option) # cookielist and ssl_engines starts with 0x400000, see also: const.py if option & 0xF00000 == 0x400000: - return slist_to_list(c_value[0]) - if c_value[0] == ffi.NULL: + return slist_to_list(self._ffi, self._lib, c_value[0]) + if c_value[0] == self._ffi.NULL: return b"" return ret_cast_option[option & 0xF00000](c_value[0]) def version(self) -> bytes: """Get the underlying libcurl version.""" - return ffi.string(lib.curl_version()) + return self._ffi.string(self._lib.curl_version()) def impersonate(self, target: str, default_headers: bool = True) -> int: """Set the browser type to impersonate. @@ -245,12 +251,12 @@ def impersonate(self, target: str, default_headers: bool = True) -> int: target: browser to impersonate. default_headers: whether to add default headers, like User-Agent. """ - return lib.curl_easy_impersonate( + return self._lib.curl_easy_impersonate( self._curl, target.encode(), int(default_headers) ) def _ensure_cacert(self): - if not self._is_cert_set: + if not self._is_cert_set and self._cacert is not None: ret = self.setopt(CurlOpt.CAINFO, self._cacert) self._check_error(ret, "set cacert") @@ -264,7 +270,7 @@ def perform(self, clear_headers: bool = True): self._ensure_cacert() # here we go - ret = lib.curl_easy_perform(self._curl) + ret = self._lib.curl_easy_perform(self._curl) try: self._check_error(ret, "perform") @@ -278,14 +284,14 @@ def clean_after_perform(self, clear_headers: bool = True): self._header_handle = None self._body_handle = None if clear_headers: - if self._headers != ffi.NULL: - lib.curl_slist_free_all(self._headers) - self._headers = ffi.NULL + if self._headers != self._ffi.NULL: + self._lib.curl_slist_free_all(self._headers) + self._headers = self._ffi.NULL def duphandle(self): """This is not a full copy of entire curl object in python. For example, headers handle is not copied, you have to set them again.""" - new_handle = lib.curl_easy_duphandle(self._curl) + new_handle = self._lib.curl_easy_duphandle(self._curl) c = Curl(cacert=self._cacert, debug=self._debug, handle=new_handle) return c @@ -293,9 +299,9 @@ def reset(self): """Reset all curl options, wrapper for curl_easy_reset.""" self._is_cert_set = False if self._curl is not None: - lib.curl_easy_reset(self._curl) + self._lib.curl_easy_reset(self._curl) self._set_error_buffer() - self._resolve = ffi.NULL + self._resolve = self._ffi.NULL def parse_cookie_headers(self, headers: List[bytes]) -> SimpleCookie: """Extract cookies.SimpleCookie from header lines. @@ -340,7 +346,23 @@ def parse_status_line(status_line: bytes) -> Tuple[CurlHttpVersion, int, bytes]: def close(self): """Close and cleanup curl handle, wrapper for curl_easy_cleanup""" if self._curl: - lib.curl_easy_cleanup(self._curl) + self._lib.curl_easy_cleanup(self._curl) self._curl = None - ffi.release(self._error_buffer) - self._resolve = ffi.NULL + self._ffi.release(self._error_buffer) + self._resolve = self._ffi.NULL + + +class CurlChrome(Curl): + # the base implementation of Curl uses Chrome + pass + + +class CurlFirefox(Curl): + + def __init__(self, debug: bool = False, handle = None): + """ + Parameters: + cacert: CA cert path to use, by default, curl_cffi uses its own bundled cert. + debug: whether to show curl debug messages. + """ + super().__init__(cacert=None, debug=debug, handle=handle, _ffi=ffi_ff, _lib=lib_ff) diff --git a/curl_cffi/requests/__init__.py b/curl_cffi/requests/__init__.py index 24beb15f..1b74875b 100644 --- a/curl_cffi/requests/__init__.py +++ b/curl_cffi/requests/__init__.py @@ -21,6 +21,7 @@ from io import BytesIO from typing import Callable, Dict, Optional, Tuple, Union +from ..curl import Curl from ..const import CurlHttpVersion from .cookies import Cookies, CookieTypes from .models import Request, Response @@ -56,6 +57,7 @@ def request( http_version: Optional[CurlHttpVersion] = None, debug: bool = False, interface: Optional[str] = None, + curl: Optional[Curl] = None ) -> Response: """Send an http request. @@ -88,7 +90,7 @@ def request( Returns: A [Response](/api/curl_cffi.requests#curl_cffi.requests.Response) object. """ - with Session(thread=thread, curl_options=curl_options, debug=debug) as s: + with Session(thread=thread, curl_options=curl_options, debug=debug, curl=curl) as s: return s.request( method=method, url=url, diff --git a/curl_cffi/requests/session.py b/curl_cffi/requests/session.py index 29418f69..aa849d85 100644 --- a/curl_cffi/requests/session.py +++ b/curl_cffi/requests/session.py @@ -13,7 +13,7 @@ from concurrent.futures import ThreadPoolExecutor -from .. import AsyncCurl, Curl, CurlError, CurlInfo, CurlOpt, CurlHttpVersion +from .. import AsyncCurl, Curl, CurlFirefox, CurlError, CurlInfo, CurlOpt, CurlHttpVersion from ..curl import CURL_WRITEFUNC_ERROR from .cookies import Cookies, CookieTypes, CurlMorsel from .errors import RequestsError @@ -43,6 +43,13 @@ class BrowserType(str, Enum): chrome99_android = "chrome99_android" safari15_3 = "safari15_3" safari15_5 = "safari15_5" + ff91esr = "ff91esr" + ff95 = "ff95" + ff98 = "ff98" + f100 = "ff100" + ff102 = "ff102" + ff109 = "ff109" + ff117 = "ff117" @classmethod def has(cls, item): @@ -373,6 +380,9 @@ def _set_curl_options( if impersonate: if not BrowserType.has(impersonate): raise RequestsError(f"impersonate {impersonate} is not supported") + if impersonate.startswith("ff"): + if not isinstance(c, CurlFirefox): + raise RequestsError(f"CurlFirefox required to impersonate {impersonate}, got {type(c)}") c.impersonate(impersonate, default_headers=default_headers) # http_version, after impersonate, which will change this to http2