Merge pull request #2866 from seleniumbase/big-uc-mode-updates-and-more

Lots of UC Mode improvements, and more

examples/raw_antibot_login.py

@@ -0,0 +1,15 @@
+"""UC Mode has PyAutoGUI methods for CAPTCHA-bypass."""
+from seleniumbase import SB
+
+with SB(uc=True, test=True) as sb:
+    url = "https://seleniumbase.io/antibot/login"
+    sb.uc_open_with_disconnect(url, 2.15)
+    sb.uc_gui_write("\t" + "demo_user")
+    sb.uc_gui_write("\t" + "secret_pass")
+    sb.uc_gui_press_keys("\t" + " ")  # For Single-char keys
+    sb.sleep(1.5)
+    sb.uc_gui_press_keys(["\t", "ENTER"])  # Multi-char keys
+    sb.reconnect(1.8)
+    sb.assert_text("Welcome!", "h1")
+    sb.set_messenger_theme(location="bottom_center")
+    sb.post_message("SeleniumBase wasn't detected!")

examples/raw_block.py

@@ -0,0 +1,11 @@
+"""If Brotector catches you, Gandalf blocks you!"""
+from seleniumbase import SB
+
+with SB(test=True) as sb:
+    url = "https://seleniumbase.io/hobbit/login"
+    sb.open(url)
+    sb.click_if_visible("button")
+    sb.assert_text("Gandalf blocked you!", "h1")
+    sb.click("img")
+    sb.highlight("h1")
+    sb.sleep(3)  # Gandalf: "You Shall Not Pass!"

examples/raw_brotector_captcha.py

@@ -0,0 +1,9 @@
+"""UC Mode has PyAutoGUI methods for CAPTCHA-bypass."""
+from seleniumbase import SB
+
+with SB(uc=True, test=True) as sb:
+    url = "https://seleniumbase.io/apps/brotector"
+    sb.uc_open_with_disconnect(url, 2.2)
+    sb.uc_gui_press_key("\t")
+    sb.uc_gui_press_key(" ")
+    sb.reconnect(2.2)

examples/raw_detection.py

@@ -0,0 +1,12 @@
+"""The Brotector CAPTCHA in action."""
+from seleniumbase import SB
+
+with SB(test=True) as sb:
+    sb.open("https://seleniumbase.io/antibot/login")
+    sb.highlight("h4", loops=6)
+    sb.type("#username", "demo_user")
+    sb.type("#password", "secret_pass")
+    sb.click_if_visible("button span")
+    sb.highlight("label#pText")
+    sb.highlight("table#detections")
+    sb.sleep(4.4)  # Add time to read the table

examples/raw_hobbit.py

@@ -0,0 +1,13 @@
+"""UC Mode has PyAutoGUI methods for CAPTCHA-bypass."""
+from seleniumbase import SB
+
+with SB(uc=True, test=True) as sb:
+    url = "https://seleniumbase.io/hobbit/login"
+    sb.uc_open_with_disconnect(url, 2.2)
+    sb.uc_gui_press_keys("\t ")
+    sb.reconnect(1.5)
+    sb.assert_text("Welcome to Middle Earth!", "h1")
+    sb.set_messenger_theme(location="bottom_center")
+    sb.post_message("SeleniumBase wasn't detected!")
+    sb.click("img")
+    sb.sleep(5.888)  # Cool animation happening now!

examples/raw_pyautogui.py

@@ -0,0 +1,19 @@
+"""
+UC Mode now has uc_gui_handle_cf(), which uses PyAutoGUI.
+An incomplete UserAgent is used to force CAPTCHA-solving.
+"""
+import sys
+from seleniumbase import SB
+
+agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/126.0.0.0"
+if "linux" in sys.platform:
+    agent = None  # Use the default UserAgent
+
+with SB(uc=True, test=True, rtf=True, agent=agent) as sb:
+    url = "https://www.virtualmanager.com/en/login"
+    sb.uc_open_with_reconnect(url, 4)
+    sb.uc_gui_handle_cf()  # Ready if needed!
+    sb.assert_element('input[name*="email"]')
+    sb.assert_element('input[name*="login"]')
+    sb.set_messenger_theme(location="bottom_center")
+    sb.post_message("SeleniumBase wasn't detected!")

help_docs/method_summary.md

@@ -111,7 +111,7 @@ self.click_partial_link(partial_link_text, timeout=None)
 # Duplicates:
 # self.click_partial_link_text(partial_link_text, timeout=None)
 
-self.get_text(selector, by="css selector", timeout=None)
+self.get_text(selector="html", by="css selector", timeout=None)
 
 self.get_attribute(selector, attribute, by="css selector", timeout=None, hard_fail=True)
 
@@ -127,7 +127,7 @@ self.remove_attributes(selector, attribute, by="css selector")
 
 self.get_property(selector, property, by="css selector", timeout=None)
 
-self.get_text_content(selector, by="css selector", timeout=None)
+self.get_text_content(selector="html", by="css selector", timeout=None)
 
 self.get_property_value(selector, property, by="css selector", timeout=None)
 
@@ -229,7 +229,7 @@ self.set_window_size(width, height)
 
 self.maximize_window()
 
-self.switch_to_frame(frame, timeout=None)
+self.switch_to_frame(frame="iframe", timeout=None)
 
 self.switch_to_default_content()
 
@@ -1032,7 +1032,7 @@ driver.get_page_source()
 
 driver.get_title()
 
-driver.switch_to_frame(frame)
+driver.switch_to_frame(frame="iframe")
 
 ############
 
@@ -1046,7 +1046,7 @@ driver.uc_open_with_tab(url)  # (New tab with default reconnect_time)
 
 driver.uc_open_with_reconnect(url, reconnect_time=None)  # (New tab)
 
-driver.uc_open_with_disconnect(url)  # Open in new tab + disconnect()
+driver.uc_open_with_disconnect(url, timeout=None)  # New tab + sleep()
 
 driver.reconnect(timeout)  # disconnect() + sleep(timeout) + connect()
 
@@ -1056,7 +1056,15 @@ driver.connect()  # Starts the webdriver service to allow actions again
 
 driver.uc_click(selector)  # A stealthy click for evading bot-detection
 
-driver.uc_switch_to_frame(frame)  # switch_to_frame() in a stealthy way
+driver.uc_gui_press_key(key)  # Use PyAutoGUI to press the keyboard key
+
+driver.uc_gui_press_keys(keys)  # Use PyAutoGUI to press a list of keys
+
+driver.uc_gui_write(text)  # Similar to uc_gui_press_keys(), but faster
+
+driver.uc_gui_handle_cf(frame="iframe")  # PyAutoGUI click CF Turnstile
+
+driver.uc_switch_to_frame(frame="iframe")  # Stealthy switch_to_frame()
 ```
 
 --------

help_docs/uc_mode.md

@@ -159,7 +159,7 @@ driver.uc_open_with_tab(url)
 
 driver.uc_open_with_reconnect(url, reconnect_time=None)
 
-driver.uc_open_with_disconnect(url)
+driver.uc_open_with_disconnect(url, timeout=None)
 
 driver.reconnect(timeout)
 
@@ -171,6 +171,14 @@ driver.uc_click(
     selector, by="css selector",
     timeout=settings.SMALL_TIMEOUT, reconnect_time=None)
 
+driver.uc_gui_press_key(key)
+
+driver.uc_gui_press_keys(keys)
+
+driver.uc_gui_write(text)
+
+driver.uc_gui_handle_cf(frame="iframe")
+
 driver.uc_switch_to_frame(frame, reconnect_time=None)
 ```
 
@@ -211,6 +219,8 @@ driver.reconnect("breakpoint")
 <li>Timing. (<b translate="no">UC Mode</b> methods let you customize default values that aren't good enough for your environment.)</li>
 <li>Not using <b><code translate="no">driver.uc_click(selector)</code></b> when you need to remain undetected while clicking something.</li>
 
+👤 On Linux, you may need to use `driver.uc_gui_handle_cf()` to successfully bypass a Cloudflare CAPTCHA. If there's more than one iframe on that website (and Cloudflare isn't the first one) then put the CSS Selector of that iframe as the first arg to `driver.uc_gui_handle_cf()`. This method uses `pyautogui`. In order for `pyautogui` to focus on the correct element, use `xvfb=True` / `--xvfb` to activate a special virtual display on Linux.
+
 👤 To find out if <b translate="no">UC Mode</b> will work at all on a specific site (before adjusting for timing), load your site with the following script:
 
 ```python

mkdocs_build/requirements.txt

@@ -3,7 +3,7 @@
 
 regex>=2024.5.15
 pymdown-extensions>=10.8.1
-pipdeptree>=2.22.0
+pipdeptree>=2.23.0
 python-dateutil>=2.8.2
 Markdown==3.6
 markdown2==2.4.13
@@ -12,15 +12,15 @@ Jinja2==3.1.4
 click==8.1.7
 ghp-import==2.1.0
 watchdog==4.0.1
-cairocffi==1.7.0
+cairocffi==1.7.1
 pathspec==0.12.1
 Babel==2.15.0
 paginate==0.5.6
 lxml==5.2.2
 pyquery==2.0.0
 readtime==3.0.0
 mkdocs==1.6.0
-mkdocs-material==9.5.26
+mkdocs-material==9.5.27
 mkdocs-exclude-search==0.6.6
 mkdocs-simple-hooks==0.1.5
 mkdocs-material-extensions==1.3.1

requirements.txt

@@ -1,18 +1,20 @@
-pip>=24.0
-packaging>=24.0
+pip>=24.0;python_version<"3.8"
+pip>=24.1;python_version>="3.8"
+packaging>=24.0;python_version<"3.8"
+packaging>=24.1;python_version>="3.8"
 setuptools>=68.0.0;python_version<"3.8"
-setuptools>=70.0.0;python_version>="3.8"
+setuptools>=70.1.0;python_version>="3.8"
 wheel>=0.42.0;python_version<"3.8"
 wheel>=0.43.0;python_version>="3.8"
 attrs>=23.2.0
 certifi>=2024.6.2
 exceptiongroup>=1.2.1
 filelock>=3.12.2;python_version<"3.8"
-filelock>=3.14.0;python_version>="3.8"
+filelock>=3.15.4;python_version>="3.8"
 platformdirs>=4.0.0;python_version<"3.8"
 platformdirs>=4.2.2;python_version>="3.8"
 typing-extensions>=4.12.2;python_version>="3.8"
-parse>=1.20.1
+parse>=1.20.2
 parse-type>=0.6.2
 pyyaml>=6.0.1
 six==1.16.0
@@ -30,8 +32,9 @@ trio==0.22.2;python_version<"3.8"
 trio==0.25.1;python_version>="3.8"
 trio-websocket==0.11.1
 wsproto==1.2.0
+websocket-client==1.8.0;python_version>="3.8"
 selenium==4.11.2;python_version<"3.8"
-selenium==4.21.0;python_version>="3.8"
+selenium==4.22.0;python_version>="3.8"
 cssselect==1.2.0
 sortedcontainers==2.4.0
 fasteners==0.19
@@ -64,6 +67,7 @@ tabcompleter==1.3.0
 pdbp==1.5.0
 colorama==0.4.6
 pyotp==2.9.0
+python-xlib==0.33;platform_system=="Linux"
 markdown-it-py==2.2.0;python_version<"3.8"
 markdown-it-py==3.0.0;python_version>="3.8"
 mdurl==0.1.2
@@ -73,13 +77,13 @@ rich==13.7.1
 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.)
 
 coverage==7.2.7;python_version<"3.8"
-coverage>=7.5.3;python_version>="3.8"
+coverage>=7.5.4;python_version>="3.8"
 pytest-cov==4.1.0;python_version<"3.8"
 pytest-cov>=5.0.0;python_version>="3.8"
 flake8==5.0.4;python_version<"3.9"
-flake8==7.0.0;python_version>="3.9"
+flake8==7.1.0;python_version>="3.9"
 mccabe==0.7.0
 pyflakes==2.5.0;python_version<"3.9"
 pyflakes==3.2.0;python_version>="3.9"
 pycodestyle==2.9.1;python_version<"3.9"
-pycodestyle==2.11.1;python_version>="3.9"
+pycodestyle==2.12.0;python_version>="3.9"

seleniumbase/__init__.py

@@ -1,4 +1,5 @@
 import collections
+import os
 import pdb
 try:
     import pdbp  # (Pdb+) --- Python Debugger Plus
@@ -34,11 +35,13 @@
         pdb.DefaultConfig.truncate_long_lines = False
         pdb.DefaultConfig.sticky_by_default = True
 colored_traceback.add_hook()
+os.environ["SE_AVOID_STATS"] = "true"  # Disable Selenium Manager stats
 if sys.version_info >= (3, 7):
     webdriver.TouchActions = None  # Lifeline for past selenium-wire versions
 if sys.version_info >= (3, 10):
     collections.Callable = collections.abc.Callable  # Lifeline for nosetests
 del collections  # Undo "import collections" / Simplify "dir(seleniumbase)"
+del os  # Undo "import os" / Simplify "dir(seleniumbase)"
 del sys  # Undo "import sys" / Simplify "dir(seleniumbase)"
 del webdriver  # Undo "import webdriver" / Simplify "dir(seleniumbase)"
 

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.27.5"
+__version__ = "4.28.0"

seleniumbase/behave/behave_sb.py

@@ -862,14 +862,17 @@ def get_configured_sb(context):
         and not sb.headless2
         and not sb.xvfb
     ):
-        print(
-            '(Linux uses "-D headless" by default. '
-            'To override, use "-D headed" / "-D gui". '
-            'For Xvfb mode instead, use "-D xvfb". '
-            "Or you can hide this info by using"
-            '"-D headless" / "-D headless2".)'
-        )
-        sb.headless = True
+        if not sb.undetectable:
+            print(
+                '(Linux uses "-D headless" by default. '
+                'To override, use "-D headed" / "-D gui". '
+                'For Xvfb mode instead, use "-D xvfb". '
+                "Or you can hide this info by using"
+                '"-D headless" / "-D headless2" / "-D uc".)'
+            )
+            sb.headless = True
+        else:
+            sb.xvfb = True
     # Recorder Mode can still optimize scripts in --headless2 mode.
     if sb.recorder_mode and sb.headless:
         sb.headless = False