From 04719e50309be94ee74b6569998dde8445036e69 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 9 Jan 2025 15:00:40 -0500 Subject: [PATCH 1/5] Update CDP Mode --- seleniumbase/core/browser_launcher.py | 24 +++++++++++++++--- seleniumbase/fixtures/base_case.py | 25 ++++++++++++++++--- seleniumbase/plugins/sb_manager.py | 13 ++++++++++ .../undetected/cdp_driver/cdp_util.py | 3 +++ 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index eb50a898daf..56439970876 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -533,10 +533,26 @@ def uc_open_with_cdp_mode(driver, url=None): if url_protocol not in ["about", "data", "chrome"]: safe_url = False + headless = False + headed = None + xvfb = None + if hasattr(sb_config, "headless"): + headless = sb_config.headless + if hasattr(sb_config, "headed"): + headed = sb_config.headed + if hasattr(sb_config, "xvfb"): + xvfb = sb_config.xvfb + loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) driver.cdp_base = loop.run_until_complete( - cdp_util.start(host=cdp_host, port=cdp_port) + cdp_util.start( + host=cdp_host, + port=cdp_port, + headless=headless, + headed=headed, + xvfb=xvfb, + ) ) loop.run_until_complete(driver.cdp_base.wait(0)) @@ -863,13 +879,15 @@ def __install_pyautogui_if_missing(): xvfb_height = 768 sb_config._xvfb_height = xvfb_height with suppress(Exception): - xvfb_display = Display( + _xvfb_display = Display( visible=True, size=(xvfb_width, xvfb_height), backend="xvfb", use_xauth=True, ) - xvfb_display.start() + _xvfb_display.start() + sb_config._virtual_display = _xvfb_display + sb_config.headless_active = True def install_pyautogui_if_missing(driver): diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index e8092ba0583..b34e3c7d5bd 100644 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -13795,7 +13795,8 @@ def __switch_to_newest_window_if_not_blank(self): if self.get_current_url() == "about:blank": self.switch_to_window(current_window) except Exception: - self.switch_to_window(current_window) + with suppress(Exception): + self.switch_to_window(current_window) def __needs_minimum_wait(self): if ( @@ -14004,9 +14005,10 @@ def __activate_standard_virtual_display(self): visible=0, size=(width, height) ) self._xvfb_display.start() - sb_config._virtual_display = self._xvfb_display self.headless_active = True - sb_config.headless_active = True + if not self.undetectable: + sb_config._virtual_display = self._xvfb_display + sb_config.headless_active = True def __activate_virtual_display(self): if self.undetectable and not (self.headless or self.headless2): @@ -14029,6 +14031,8 @@ def __activate_virtual_display(self): "\nX11 display failed! Will use regular xvfb!" ) self.__activate_standard_virtual_display() + else: + self.headless_active = True except Exception as e: if hasattr(e, "msg"): print("\n" + str(e.msg)) @@ -16601,6 +16605,7 @@ def tearDown(self): self.__quit_all_drivers() # Resume tearDown() for all test runners, (Pytest / Pynose / Behave) if hasattr(self, "_xvfb_display") and self._xvfb_display: + # Stop the Xvfb virtual display launched from BaseCase try: if hasattr(self._xvfb_display, "stop"): self._xvfb_display.stop() @@ -16610,6 +16615,20 @@ def tearDown(self): pass except Exception: pass + if ( + hasattr(sb_config, "_virtual_display") + and sb_config._virtual_display + and hasattr(sb_config._virtual_display, "stop") + ): + # CDP Mode may launch a 2nd Xvfb virtual display + try: + sb_config._virtual_display.stop() + sb_config._virtual_display = None + sb_config.headless_active = False + except AttributeError: + pass + except Exception: + pass if self.__visual_baseline_copies: sb_config._visual_baseline_copies = True if has_exception: diff --git a/seleniumbase/plugins/sb_manager.py b/seleniumbase/plugins/sb_manager.py index 3c39eca7c15..96f916cf2e0 100644 --- a/seleniumbase/plugins/sb_manager.py +++ b/seleniumbase/plugins/sb_manager.py @@ -1256,6 +1256,19 @@ def SB( print(traceback.format_exc().strip()) if test and not test_passed: print("********** ERROR: The test AND the tearDown() FAILED!") + if ( + hasattr(sb_config, "_virtual_display") + and sb_config._virtual_display + and hasattr(sb_config._virtual_display, "stop") + ): + try: + sb_config._virtual_display.stop() + sb_config._virtual_display = None + sb_config.headless_active = False + except AttributeError: + pass + except Exception: + pass end_time = time.time() run_time = end_time - start_time sb_config = sb_config_backup diff --git a/seleniumbase/undetected/cdp_driver/cdp_util.py b/seleniumbase/undetected/cdp_driver/cdp_util.py index 7899436f2eb..237ba3b28b3 100644 --- a/seleniumbase/undetected/cdp_driver/cdp_util.py +++ b/seleniumbase/undetected/cdp_driver/cdp_util.py @@ -84,6 +84,9 @@ def __activate_virtual_display_as_needed( "\nX11 display failed! Will use regular xvfb!" ) __activate_standard_virtual_display() + else: + sb_config._virtual_display = _xvfb_display + sb_config.headless_active = True except Exception as e: if hasattr(e, "msg"): print("\n" + str(e.msg)) From cd38f864f9839fd2add059607a9d945326c7dd4e Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 9 Jan 2025 15:12:01 -0500 Subject: [PATCH 2/5] Update examples --- examples/cdp_mode/ReadMe.md | 7 +- examples/presenter/hacking_with_cdp.py | 146 +++++++++++++++++++++++++ examples/raw_skype_mobile.py | 15 +++ 3 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 examples/presenter/hacking_with_cdp.py create mode 100644 examples/raw_skype_mobile.py diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index 95e3aa6a6ea..60d204a0eb0 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -6,11 +6,16 @@ -------- - +

(Watch the CDP Mode tutorial on YouTube! ▶️)

-------- + +

(Watch "Hacking websites with CDP" on YouTube! ▶️)

+ +-------- + 👤 UC Mode avoids bot-detection by first disconnecting WebDriver from the browser at strategic times, calling special PyAutoGUI methods to bypass CAPTCHAs (as needed), and finally reconnecting the driver afterwards so that WebDriver actions can be performed again. Although this approach works for bypassing simple CAPTCHAs, more flexibility is needed for bypassing bot-detection on websites with advanced protection. (That's where CDP Mode comes in.) 🐙 CDP Mode is based on python-cdp, trio-cdp, and nodriver. trio-cdp is an early implementation of python-cdp, and nodriver is a modern implementation of python-cdp. (Refactored Python-CDP code is imported from MyCDP.) diff --git a/examples/presenter/hacking_with_cdp.py b/examples/presenter/hacking_with_cdp.py new file mode 100644 index 00000000000..0e5341f11fa --- /dev/null +++ b/examples/presenter/hacking_with_cdp.py @@ -0,0 +1,146 @@ +# https://www.youtube.com/watch?v=vt2zsdiNh3U +from seleniumbase import BaseCase +BaseCase.main(__name__, __file__) + + +class UCPresentationClass(BaseCase): + def test_hacking_with_cdp(self): + self.open("data:,") + self.set_window_position(4, 40) + self._output_file_saves = False + self.create_presentation(theme="serif", transition="none") + self.add_slide("

Press SPACE to begin!

\n") + self.add_slide( + "

Coming up on the Hacker Show...

\n" + "
", + ) + self.add_slide( + "

Coming up on the Hacker Show...

\n" + "

", + ) + self.add_slide( + "

Coming up on the Hacker Show...

\n" + "

", + ) + self.add_slide( + "

Coming up on the Hacker Show...

\n" + "

", + ) + self.add_slide( + "

Coming up on the Hacker Show...

\n" + "

", + ) + self.add_slide( + "

Coming up on the Hacker Show...

\n" + "

", + ) + self.add_slide( + "

Get ready for some
serious hacking!

" + ) + self.add_slide( + '' + ) + self.add_slide( + '' + ) + self.add_slide( + '' + ) + self.add_slide( + '' + ) + self.add_slide( + '' + ) + self.add_slide( + '' + ) + self.add_slide( + '' + ) + self.add_slide( + '' + ) + self.add_slide( + '' + ) + self.add_slide( + '' + ) + self.add_slide( + '' + ) + self.add_slide( + '' + ) + self.add_slide( + '

The remote-debugging-port

' + '' + ) + self.add_slide( + "

Let's get to the fun part...

" + '' + ) + self.begin_presentation(filename="uc_presentation.html") diff --git a/examples/raw_skype_mobile.py b/examples/raw_skype_mobile.py new file mode 100644 index 00000000000..2f4ef2c6a6c --- /dev/null +++ b/examples/raw_skype_mobile.py @@ -0,0 +1,15 @@ +"""Mobile emulation test for Skype.""" +from seleniumbase import SB + +with SB(mobile=True, test=True) as sb: + sb.open("https://www.skype.com/en/get-skype/") + sb.assert_element('[aria-label="Microsoft"]') + sb.assert_text("Download Skype", "h1") + sb.highlight("div.appBannerContent") + sb.highlight("h1") + sb.assert_text("Skype for Mobile", "h2") + sb.highlight("h2") + sb.highlight("#get-skype-0") + sb.highlight_click("span[data-dropdown-icon]") + sb.highlight("#get-skype-0_android-download") + sb.highlight('[data-bi-id*="ios"]') From 78b180ffb2aadd7e259f1b2ebafade811409c941 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 9 Jan 2025 15:12:47 -0500 Subject: [PATCH 3/5] Refresh Python dependencies --- mkdocs_build/requirements.txt | 2 +- requirements.txt | 9 +++++---- setup.py | 11 ++++++----- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/mkdocs_build/requirements.txt b/mkdocs_build/requirements.txt index 4608be4cd1e..4379c934aae 100644 --- a/mkdocs_build/requirements.txt +++ b/mkdocs_build/requirements.txt @@ -2,7 +2,7 @@ # Minimum Python version: 3.9 (for generating docs only) regex>=2024.11.6 -pymdown-extensions>=10.13 +pymdown-extensions>=10.14 pipdeptree>=2.24.0 python-dateutil>=2.8.2 Markdown==3.7 diff --git a/requirements.txt b/requirements.txt index 806752a8c0d..b0b6de70735 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ pip>=24.3.1 packaging>=24.2 setuptools~=70.2;python_version<"3.10" -setuptools>=75.6.0;python_version>="3.10" +setuptools>=75.8.0;python_version>="3.10" wheel>=0.45.1 attrs>=24.3.0 certifi>=2024.12.14 @@ -23,7 +23,7 @@ parse>=1.20.2 parse-type>=0.6.4 colorama>=0.4.6 pyyaml>=6.0.2 -pygments>=2.18.0 +pygments>=2.19.1 pyreadline3>=3.5.3;platform_system=="Windows" tabcompleter>=1.4.0 pdbp>=1.6.1 @@ -36,7 +36,8 @@ requests==2.32.3 sniffio==1.3.1 h11==0.14.0 outcome==1.3.0.post0 -trio==0.27.0 +trio==0.27.0;python_version<"3.9" +trio==0.28.0;python_version>="3.9" trio-websocket==0.11.1 wsproto==1.2.0 websocket-client==1.8.0 @@ -67,7 +68,7 @@ rich==13.9.4 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.) coverage>=7.6.1;python_version<"3.9" -coverage>=7.6.9;python_version>="3.9" +coverage>=7.6.10;python_version>="3.9" pytest-cov>=5.0.0;python_version<"3.9" pytest-cov>=6.0.0;python_version>="3.9" flake8==5.0.4;python_version<"3.9" diff --git a/setup.py b/setup.py index 01615ed533e..c9308093c1e 100755 --- a/setup.py +++ b/setup.py @@ -150,7 +150,7 @@ 'pip>=24.3.1', 'packaging>=24.2', 'setuptools~=70.2;python_version<"3.10"', # Newer ones had issues - 'setuptools>=75.6.0;python_version>="3.10"', + 'setuptools>=75.8.0;python_version>="3.10"', 'wheel>=0.45.1', 'attrs>=24.3.0', "certifi>=2024.12.14", @@ -172,7 +172,7 @@ 'parse-type>=0.6.4', 'colorama>=0.4.6', 'pyyaml>=6.0.2', - 'pygments>=2.18.0', + 'pygments>=2.19.1', 'pyreadline3>=3.5.3;platform_system=="Windows"', "tabcompleter>=1.4.0", "pdbp>=1.6.1", @@ -185,7 +185,8 @@ 'sniffio==1.3.1', 'h11==0.14.0', 'outcome==1.3.0.post0', - 'trio==0.27.0', + 'trio==0.27.0;python_version<"3.9"', + 'trio==0.28.0;python_version>="3.9"', 'trio-websocket==0.11.1', 'wsproto==1.2.0', 'websocket-client==1.8.0', @@ -225,7 +226,7 @@ # Usage: coverage run -m pytest; coverage html; coverage report "coverage": [ 'coverage>=7.6.1;python_version<"3.9"', - 'coverage>=7.6.9;python_version>="3.9"', + 'coverage>=7.6.10;python_version>="3.9"', 'pytest-cov>=5.0.0;python_version<"3.9"', 'pytest-cov>=6.0.0;python_version>="3.9"', ], @@ -264,7 +265,7 @@ # (An optional library for image-processing.) "pillow": [ 'Pillow>=10.4.0;python_version<"3.9"', - 'Pillow>=11.0.0;python_version>="3.9"', + 'Pillow>=11.1.0;python_version>="3.9"', ], # pip install -e .[pip-system-certs] # (If you see [SSL: CERTIFICATE_VERIFY_FAILED], then get this.) From 1caf2feb4af08537f862819db95a5983e4a0250d Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 9 Jan 2025 15:13:11 -0500 Subject: [PATCH 4/5] Welcome to 2025 --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index a7133a96b68..8805a8182c6 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2014-2024 Michael Mintz +Copyright (c) 2014-2025 Michael Mintz Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 4ef08127ae832a6d2e77d82404a462b55d834fa7 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 9 Jan 2025 15:13:35 -0500 Subject: [PATCH 5/5] Version 4.33.13 --- seleniumbase/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 5a2eda19a6a..634fe5c2052 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.33.12" +__version__ = "4.33.13"