Merge pull request #3117 from seleniumbase/stealthier-recorder-and-more

Stealthier Recorder and more

README.md

@@ -14,7 +14,7 @@
 
 <p align="center" class="hero__title"><b>All-in-one Browser Automation Framework:<br />Web Crawling / Testing / Scraping / Stealth</b></p>
 
-<p align="center"><a href="https://pypi.python.org/pypi/seleniumbase" target="_blank"><img src="https://img.shields.io/pypi/v/seleniumbase.svg?color=3399EE" alt="PyPI version" /></a> <a href="https://github.com/seleniumbase/SeleniumBase/releases" target="_blank"><img src="https://img.shields.io/github/v/release/seleniumbase/SeleniumBase.svg?color=22AAEE" alt="GitHub version" /></a> <a href="https://seleniumbase.io"><img src="https://img.shields.io/badge/docs-seleniumbase.io-11BBAA.svg" alt="SeleniumBase Docs" /></a> <a href="https://github.com/seleniumbase/SeleniumBase/actions" target="_blank"><img src="https://github.com/seleniumbase/SeleniumBase/workflows/CI%20build/badge.svg" alt="SeleniumBase GitHub Actions" /></a> <a href="https://gitter.im/seleniumbase/SeleniumBase" target="_blank"><img src="https://img.shields.io/gitter/room/seleniumbase/SeleniumBase.svg" alt="Gitter chat"/></a></p>
+<p align="center"><a href="https://pypi.python.org/pypi/seleniumbase" target="_blank"><img src="https://img.shields.io/pypi/v/seleniumbase.svg?color=3399EE" alt="PyPI version" /></a> <a href="https://github.com/seleniumbase/SeleniumBase/releases" target="_blank"><img src="https://img.shields.io/github/v/release/seleniumbase/SeleniumBase.svg?color=22AAEE" alt="GitHub version" /></a> <a href="https://seleniumbase.io"><img src="https://img.shields.io/badge/docs-seleniumbase.io-11BBAA.svg" alt="SeleniumBase Docs" /></a> <a href="https://github.com/seleniumbase/SeleniumBase/actions" target="_blank"><img src="https://github.com/seleniumbase/SeleniumBase/workflows/CI%20build/badge.svg" alt="SeleniumBase GitHub Actions" /></a> <a href="https://discord.gg/EdhQTn3EyE" target="_blank"><img src="https://img.shields.io/badge/join-discord-infomational" alt="Join the SeleniumBase chat on Discord"/></a></p>
 
 <p align="center">
 <a href="#python_installation">🚀 Start</a> |
@@ -1361,22 +1361,23 @@ pytest --reruns=1 --reruns-delay=1
 <div><a href="https://github.com/seleniumbase/SeleniumBase/issues?q=is%3Aissue+is%3Aclosed"><img src="https://img.shields.io/github/issues-closed-raw/seleniumbase/SeleniumBase.svg?color=22BB88" title="Closed Issues" /></a> <a href="https://github.com/seleniumbase/SeleniumBase/pulls?q=is%3Apr+is%3Aclosed"><img src="https://img.shields.io/github/issues-pr-closed/seleniumbase/SeleniumBase.svg?logo=github&logoColor=white&color=22BB99" title="Closed Pull Requests" /></a></div>
 </p>
 
-<p align="left"><a href="https://github.com/seleniumbase/SeleniumBase/"><img src="https://seleniumbase.github.io/cdn/img/sb_logo_10t.png" alt="SeleniumBase" title="SeleniumBase" width="266" /></a></p>
+<p align="left"><a href="https://github.com/seleniumbase/SeleniumBase/"><img src="https://seleniumbase.github.io/cdn/img/sb_logo_10t.png" alt="SeleniumBase" title="SeleniumBase" width="274" /></a></p>
 
 <a href="https://pypi.org/project/seleniumbase/" target="_blank"><img src="https://img.shields.io/pypi/pyversions/seleniumbase.svg?color=22AAEE&logo=python&logoColor=FEDC54" title="Supported Python Versions" /></a>
 
 <p><div>
-<span><a href="https://www.youtube.com/playlist?list=PLp9uKicxkBc5UIlGi2BuE3aWC7JyXpD3m"><img src="https://seleniumbase.github.io/cdn/img/youtube.png" title="SeleniumBase Playlist on YouTube" alt="SeleniumBase Playlist on YouTube" width="68" /></a></span>
-<span><a href="https://github.com/seleniumbase/SeleniumBase"><img src="https://seleniumbase.github.io/img/social/share_github.svg" title="SeleniumBase on GitHub" alt="SeleniumBase on GitHub" width="62" /></a></span>
-<span><a href="https://www.facebook.com/SeleniumBase"><img src="https://seleniumbase.io/img/social/share_facebook.svg" title="SeleniumBase on Facebook" alt="SeleniumBase on Facebook" width="64" /></a></span>
-<span><a href="https://gitter.im/seleniumbase/SeleniumBase" target="_blank"><img src="https://seleniumbase.github.io/img/social/share_gitter.svg" title="SeleniumBase on Gitter" alt="SeleniumBase on Gitter" width="50" /></a></span>
+<span><a href="https://www.youtube.com/playlist?list=PLp9uKicxkBc5UIlGi2BuE3aWC7JyXpD3m"><img src="https://seleniumbase.github.io/cdn/img/youtube.png" title="SeleniumBase Playlist on YouTube" alt="SeleniumBase Playlist on YouTube" width="70" /></a></span>
+<span><a href="https://github.com/seleniumbase/SeleniumBase"><img src="https://seleniumbase.github.io/img/social/share_github.svg" title="SeleniumBase on GitHub" alt="SeleniumBase on GitHub" width="64" /></a></span>
+<span><a href="https://discord.gg/EdhQTn3EyE"><img src="https://seleniumbase.github.io/other/discord_icon.png" title="SeleniumBase on Discord" alt="SeleniumBase on Discord" width="66" /></a></span>
+<span><a href="https://www.facebook.com/SeleniumBase"><img src="https://seleniumbase.io/img/social/share_facebook.svg" title="SeleniumBase on Facebook" alt="SeleniumBase on Facebook" width="62" /></a></span>
+<span><a href="https://gitter.im/seleniumbase/SeleniumBase" target="_blank"><img src="https://seleniumbase.github.io/img/social/share_gitter.svg" title="SeleniumBase on Gitter" alt="SeleniumBase on Gitter" width="48" /></a></span>
 </div></p>
 
 <p><div><b><a href="https://github.com/mdmintz">https://github.com/mdmintz</a></b></div></p>
 
-<div><a href="https://github.com/seleniumbase/SeleniumBase/"><img src="https://seleniumbase.github.io/cdn/img/super_logo_sb3.png" title="SeleniumBase" width="240" /></a></div>
-<div><a href="https://seleniumbase.io"><img src="https://img.shields.io/badge/docs-seleniumbase.io-11BBAA.svg" alt="SeleniumBase Docs" /></a></div> <div><a href="https://github.com/seleniumbase/SeleniumBase"><img src="https://img.shields.io/badge/tested%20with-SeleniumBase-04C38E.svg" alt="Tested with SeleniumBase" /></a></div> <div><a href="https://github.com/seleniumbase/SeleniumBase/blob/master/LICENSE"><img src="https://img.shields.io/badge/license-MIT-22BBCC.svg" title="SeleniumBase" /></a> <a href="https://gitter.im/seleniumbase/SeleniumBase" target="_blank"><img src="https://img.shields.io/gitter/room/seleniumbase/SeleniumBase.svg" alt="Gitter chat"/></a></div>
-<div><a href="https://hellogithub.com/repository/c6be2d0f1969448697683d11a4ff915e" target="_blank"><img src="https://abroad.hellogithub.com/v1/widgets/recommend.svg?rid=c6be2d0f1969448697683d11a4ff915e&claim_uid=xcrm4p9j3d6JCO5" alt="Featured｜HelloGitHub" style="width: 173px; height: 38px;" width="173" height="38" /></a></div>
-<div><a href="https://pepy.tech/project/seleniumbase" target="_blank"><img src="https://static.pepy.tech/badge/seleniumbase" alt="SeleniumBase PyPI downloads" /></a></div>
-<div><a href="https://github.com/seleniumbase/SeleniumBase/stargazers"><img src="https://img.shields.io/github/stars/seleniumbase/seleniumbase.svg?color=19A57B" title="Stargazers" /></a></div>
-<div align="left"><img src="https://views.whatilearened.today/views/github/seleniumbase/SeleniumBase.svg" width="124px" height="28px" alt="Views" /></div>
+<div><a href="https://github.com/seleniumbase/SeleniumBase/"><img src="https://seleniumbase.github.io/cdn/img/super_logo_sb3.png" title="SeleniumBase" width="274" /></a></div>
+<div><a href="https://seleniumbase.io"><img src="https://img.shields.io/badge/docs-seleniumbase.io-11BBAA.svg" alt="SeleniumBase Docs" /></a> <a href="https://github.com/seleniumbase/SeleniumBase/blob/master/LICENSE"><img src="https://img.shields.io/badge/license-MIT-22BBCC.svg" title="SeleniumBase" /></a></div>
+<div><a href="https://github.com/seleniumbase/SeleniumBase"><img src="https://img.shields.io/badge/tested%20with-SeleniumBase-04C38E.svg" alt="Tested with SeleniumBase" /></a> <a href="https://github.com/seleniumbase/SeleniumBase/stargazers"><img src="https://img.shields.io/github/stars/seleniumbase/seleniumbase.svg?color=19A57B" title="Stargazers" /></a></div>
+<div><a href="https://hellogithub.com/repository/c6be2d0f1969448697683d11a4ff915e" target="_blank"><img src="https://abroad.hellogithub.com/v1/widgets/recommend.svg?rid=c6be2d0f1969448697683d11a4ff915e&claim_uid=xcrm4p9j3d6JCO5&theme=small" alt="Featured｜HelloGitHub" /></a> <a href="https://discord.gg/EdhQTn3EyE" target="_blank"><img src="https://img.shields.io/badge/join-discord-infomational" alt="Join the SeleniumBase chat on Discord"/></a> <a href="https://gitter.im/seleniumbase/SeleniumBase" target="_blank"><img src="https://img.shields.io/gitter/room/seleniumbase/SeleniumBase.svg" alt="Gitter chat"/></a></div>
+<div><a href="https://pepy.tech/project/seleniumbase" target="_blank"><img src="https://static.pepy.tech/badge/seleniumbase" alt="SeleniumBase PyPI downloads" /></a> <img src="https://views.whatilearened.today/views/github/seleniumbase/SeleniumBase.svg" width="98px" height="20px" alt="Views" /></div>
+<div align="left"></div>

examples/ReadMe.md

@@ -251,6 +251,12 @@ python gui_test_runner.py
 
 --------
 
+<h3><a href="https://discord.gg/EdhQTn3EyE"><img src="https://seleniumbase.github.io/other/discord_icon.png" title="Join the SeleniumBase chat on Discord" alt="Join the SeleniumBase chat on Discord" width="44" /></a> <a href="https://discord.gg/EdhQTn3EyE">Join the SeleniumBase chat on Discord!</a></h3>
+
+Ask questions. Find answers. Learn how to automate!
+
+--------
+
 <img src="https://seleniumbase.github.io/cdn/img/super_logo_sb.png" title="SeleniumBase" width="320" />
 
 <a href="https://github.com/seleniumbase/SeleniumBase">

integrations/github/workflows/extras.md

@@ -1,10 +1,10 @@
 <h3><img src="https://seleniumbase.github.io/img/logo3a.png" title="SeleniumBase" width="32" /> Integrations for GitHub Actions:</h3>
 
 ### Uploading Artifacts:
-* Here's an example using [upload-artifact@v2](https://github.com/actions/upload-artifact) to push up a SeleniumBase-generated artifact.
+* Here's an example using [upload-artifact@v4](https://github.com/actions/upload-artifact) to push up a SeleniumBase-generated artifact.
 
-```
-    - uses: actions/upload-artifact@v2
+```yml
+    - uses: actions/upload-artifact@v4
       with:
         name: Click to download the presentation
         path: saved_presentations/my_presentation.html
@@ -18,7 +18,7 @@
 * For this particular action, ``SLACK_CHANNEL`` is an optional environment variable that defaults to the webhook token channel if not specified.
 * The following example shows how to put a link to your workflow as the ``SLACK_MESSAGE`` (Lets you see artifacts pushed up, such as from the SeleniumBase Presenter feature!):
 
-```
+```yml
     - name: Slack notification
       uses: rtCamp/action-slack-notify@master
       env:

mkdocs.yml

@@ -75,6 +75,7 @@ plugins:
         - examples/case_summary.md
         - help_docs/chinese.md
         - integrations/katalon/ReadMe.md
+        - help_docs/ReadMe.md
         - help_docs/verify_webdriver.md
         - help_docs/webdriver_installation.md
         - seleniumbase/masterqa/ReadMe.md

mkdocs_build/requirements.txt

@@ -3,7 +3,7 @@
 
 regex>=2024.7.24
 pymdown-extensions>=10.9
-pipdeptree>=2.23.1
+pipdeptree>=2.23.3
 python-dateutil>=2.8.2
 Markdown==3.7
 markdown2==2.5.0

requirements.txt

@@ -11,9 +11,9 @@ attrs>=24.2.0
 certifi>=2024.8.30
 exceptiongroup>=1.2.2
 filelock>=3.12.2;python_version<"3.8"
-filelock>=3.15.4;python_version>="3.8"
+filelock>=3.16.0;python_version>="3.8"
 platformdirs>=4.0.0;python_version<"3.8"
-platformdirs>=4.2.2;python_version>="3.8"
+platformdirs>=4.3.2;python_version>="3.8"
 typing-extensions>=4.12.2;python_version>="3.8"
 parse>=1.20.2
 parse-type>=0.6.3
@@ -47,7 +47,7 @@ pluggy==1.2.0;python_version<"3.8"
 pluggy==1.5.0;python_version>="3.8"
 py==1.11.0
 pytest==7.4.4;python_version<"3.8"
-pytest==8.3.2;python_version>="3.8"
+pytest==8.3.3;python_version>="3.8"
 pytest-html==2.0.1
 pytest-metadata==3.0.0;python_version<"3.8"
 pytest-metadata==3.1.1;python_version>="3.8"
@@ -73,7 +73,7 @@ python-xlib==0.33;platform_system=="Linux"
 markdown-it-py==2.2.0;python_version<"3.8"
 markdown-it-py==3.0.0;python_version>="3.8"
 mdurl==0.1.2
-rich==13.8.0
+rich==13.8.1
 
 # --- Testing Requirements --- #
 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.)

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.30.3"
+__version__ = "4.30.4"

seleniumbase/common/ReadMe.md

@@ -1,6 +1,6 @@
 <!-- SeleniumBase Docs -->
 
-## Using [seleniumbase/common](https://github.com/seleniumbase/SeleniumBase/blob/master/seleniumbase/common) methods.
+## [seleniumbase/common](https://github.com/seleniumbase/SeleniumBase/blob/master/seleniumbase/common) decorators and security.
 
 ### Part 1: Decorators - (from [decorators.py](https://github.com/seleniumbase/SeleniumBase/blob/master/seleniumbase/common/decorators.py))
 

seleniumbase/console_scripts/sb_mkrec.py

@@ -181,6 +181,28 @@ def main():
     data.append("")
     data.append("class RecorderTest(BaseCase):")
     data.append("    def test_recording(self):")
+    if use_uc:
+        data.append("        if self.undetectable:")
+        if (
+            start_page
+            and (
+                start_page.startswith("http:")
+                or start_page.startswith("https:")
+                or start_page.startswith("file:")
+            )
+        ):
+            used_sp = start_page
+            if '"' not in start_page:
+                used_sp = '"%s"' % start_page
+            elif "'" not in start_page:
+                used_sp = "'%s'" % start_page
+            data.append(
+                "            self.uc_open_with_disconnect(\n"
+                "                %s\n"
+                "            )" % used_sp
+            )
+        else:
+            data.append("            self.disconnect()")
     data.append("        if self.recorder_ext:")
     data.append("            # When done recording actions,")
     data.append('            # type "c", and press [Enter].')
@@ -231,7 +253,18 @@ def main():
     )
     print(success)
     run_cmd = None
-    if not start_page:
+    if (
+        not start_page
+        or (
+            use_uc
+            and (
+                start_page.startswith("http:")
+                or start_page.startswith("https:")
+                or start_page.startswith("file:")
+            )
+            and not esc_end
+        )
+    ):
         run_cmd = "%s -m pytest %s --rec -q -s" % (sys_executable, file_name)
     else:
         run_cmd = "%s -m pytest %s --rec -q -s --url=%s" % (

seleniumbase/console_scripts/sb_recorder.py

@@ -197,6 +197,13 @@ def do_playback(file_name, use_chrome, window, demo_mode=False):
         command += " --edge"
     if demo_mode:
         command += " --demo"
+    command_args = sys.argv[2:]
+    if (
+        "--uc" in command_args
+        or "--undetected" in command_args
+        or "--undetectable" in command_args
+    ):
+        command += " --uc"
     poll = None
     if sb_config.rec_subprocess_used:
         poll = sb_config.rec_subprocess_p.poll()

seleniumbase/core/browser_launcher.py

@@ -728,8 +728,23 @@ def uc_gui_click_x_y(driver, x, y, timeframe=0.25):
         install_pyautogui_if_missing(driver)
         import pyautogui
         pyautogui = get_configured_pyautogui(pyautogui)
+        connected = True
+        width_ratio = 1.0
         if IS_WINDOWS:
-            width_ratio = 1.0
+            try:
+                driver.window_handles
+            except Exception:
+                connected = False
+            if (
+                not connected
+                and (
+                    not hasattr(sb_config, "_saved_width_ratio")
+                    or not sb_config._saved_width_ratio
+                )
+            ):
+                driver.reconnect(0.1)
+                connected = True
+        if IS_WINDOWS and connected:
             window_rect = driver.get_window_rect()
             width = window_rect["width"]
             height = window_rect["height"]
@@ -751,13 +766,24 @@ def uc_gui_click_x_y(driver, x, y, timeframe=0.25):
                 sb_config._saved_width_ratio = width_ratio
             driver.minimize_window()
             driver.set_window_rect(win_x, win_y, width, height)
+        elif (
+            IS_WINDOWS
+            and not connected
+            and hasattr(sb_config, "_saved_width_ratio")
+            and sb_config._saved_width_ratio
+        ):
+            width_ratio = sb_config._saved_width_ratio
+        if IS_WINDOWS:
             x = x * width_ratio
             y = y * width_ratio
             _uc_gui_click_x_y(driver, x, y, timeframe=timeframe, uc_lock=False)
             return
-        page_actions.switch_to_window(
-            driver, driver.current_window_handle, 2, uc_lock=False
-        )
+        try:
+            page_actions.switch_to_window(
+                driver, driver.current_window_handle, 2, uc_lock=False
+            )
+        except Exception:
+            pass
         _uc_gui_click_x_y(driver, x, y, timeframe=timeframe, uc_lock=False)
 
 
@@ -958,21 +984,20 @@ def _uc_gui_click_captcha(
             pass
     reconnect_time = (float(constants.UC.RECONNECT_TIME) / 2.0) + 0.5
     if IS_LINUX:
-        reconnect_time = constants.UC.RECONNECT_TIME + 0.15
+        reconnect_time = constants.UC.RECONNECT_TIME + 0.2
     if not x or not y:
         reconnect_time = 1  # Make it quick (it already failed)
     driver.reconnect(reconnect_time)
-    if blind:
+    if blind or (IS_LINUX and "Just a moment" in driver.title):
         retry = True
+        blind = True
     if retry and x and y and _on_a_captcha_page(driver):
         with gui_lock:  # Prevent issues with multiple processes
             # Make sure the window is on top
             page_actions.switch_to_window(
                 driver, driver.current_window_handle, 2, uc_lock=False
             )
-            if not driver.is_element_present("iframe"):
-                return
-            else:
+            if driver.is_element_present("iframe"):
                 try:
                     driver.switch_to_frame(frame)
                 except Exception:
@@ -1179,7 +1204,7 @@ def _uc_gui_handle_captcha(
             pass
     reconnect_time = (float(constants.UC.RECONNECT_TIME) / 2.0) + 0.5
     if IS_LINUX:
-        reconnect_time = constants.UC.RECONNECT_TIME + 0.15
+        reconnect_time = constants.UC.RECONNECT_TIME + 0.2
     driver.reconnect(reconnect_time)
 
 

seleniumbase/fixtures/base_case.py

@@ -16029,6 +16029,11 @@ def tearDown(self):
             # This test already called tearDown()
             return
         if hasattr(self, "recorder_mode") and self.recorder_mode:
+            if self.undetectable:
+                try:
+                    self.driver.window_handles
+                except Exception:
+                    self.driver.connect()
             self.__process_recorded_actions()
         self.__called_teardown = True
         self.__called_setup = False

setup.py

@@ -159,9 +159,9 @@
         "certifi>=2024.8.30",
         "exceptiongroup>=1.2.2",
         'filelock>=3.12.2;python_version<"3.8"',
-        'filelock>=3.15.4;python_version>="3.8"',
+        'filelock>=3.16.0;python_version>="3.8"',
         'platformdirs>=4.0.0;python_version<"3.8"',
-        'platformdirs>=4.2.2;python_version>="3.8"',
+        'platformdirs>=4.3.2;python_version>="3.8"',
         'typing-extensions>=4.12.2;python_version>="3.8"',
         'parse>=1.20.2',
         'parse-type>=0.6.3',
@@ -195,7 +195,7 @@
         'pluggy==1.5.0;python_version>="3.8"',
         "py==1.11.0",  # Needed by pytest-html
         'pytest==7.4.4;python_version<"3.8"',
-        'pytest==8.3.2;python_version>="3.8"',
+        'pytest==8.3.3;python_version>="3.8"',
         "pytest-html==2.0.1",  # Newer ones had issues
         'pytest-metadata==3.0.0;python_version<"3.8"',
         'pytest-metadata==3.1.1;python_version>="3.8"',
@@ -221,7 +221,7 @@
         'markdown-it-py==2.2.0;python_version<"3.8"',
         'markdown-it-py==3.0.0;python_version>="3.8"',
         'mdurl==0.1.2',
-        'rich==13.8.0',
+        'rich==13.8.1',
     ],
     extras_require={
         # pip install -e .[allure]
@@ -310,7 +310,7 @@
             'hyperframe==6.0.1',
             'kaitaistruct==0.10',
             'pyasn1==0.5.1;python_version<"3.8"',
-            'pyasn1==0.6.0;python_version>="3.8"',
+            'pyasn1==0.6.1;python_version>="3.8"',
             'zstandard==0.23.0',
         ],
     },