diff --git a/README.md b/README.md
index 00527e3a084..b2ecddd35c8 100755
--- a/README.md
+++ b/README.md
@@ -19,30 +19,30 @@
π Start |
π° Features |
-π Examples |
ποΈ Options |
+π Examples |
π Scripts |
π± Mobile
π APIs |
- π‘ Formats |
-π Dashboard |
+ π Formats |
π΄ Recorder |
+π Dashboard |
πΎ Locales |
-π Grid
+π» Farm
ποΈ GUI |
π° TestPage |
-ποΈ CasePlans |
π€ UC Mode |
-𧬠Hybrid |
-π» Farm
+π CDP Mode |
+πΆ Charts |
+π Grid
ποΈ How |
π Migrate |
-β»οΈ Templates |
-π NodeGUI |
-πΆ Charts |
+ποΈ CasePlans |
+β»οΈ Template |
+𧬠Hybrid |
π Tours
π€ CI/CD |
diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md
new file mode 100644
index 00000000000..cf9c1a360c4
--- /dev/null
+++ b/examples/cdp_mode/ReadMe.md
@@ -0,0 +1,301 @@
+
+
+## [SeleniumBase CDP Mode (Chrome Devtools Protocol Mode) is a special mode inside of SeleniumBase UC Mode CDP-Driver . Although regular UC Mode can't perform WebDriver actions while the driver is disconnected from the browser, the CDP-Driver can still perform actions (while maintaining its cover).
+
+π€ UC Mode avoids bot-detection by first disconnecting WebDriver from the browser at strategic times, calling special PyAutoGUI methods to bypass CAPTCHAs (as needed), and finally reconnecting the driver afterwards so that WebDriver actions can be performed again. Although this approach works for bypassing simple CAPTCHAs, more flexibility is needed for bypassing bot-detection on websites with advanced protection. (That's where CDP Mode comes in.)
+
+π CDP Mode is based on python-cdp , trio-cdp , and nodriver . trio-cdp was an early implementation of python-cdp, whereas nodriver is a modern implementation of python-cdp. (Refactored CDP code is imported from MyCDP .)
+
+π CDP Mode includes multiple updates to the above, such as:
+
+* Sync methods. (Using `async`/`await` is not necessary!)
+* The ability to use WebDriver and CDP-Driver together.
+* Backwards compatibility for existing UC Mode scripts.
+* More configuration options when launching browsers.
+* More methods. (And bug-fixes for existing methods.)
+* Faster response time for support. (Eg. [Discord Chat](https://discord.gg/EdhQTn3EyE))
+
+--------
+
+### π CDP Mode initialization:
+
+* `sb.activate_cdp_mode(url)`
+
+> (Call that from a **UC Mode** script)
+
+--------
+
+### π CDP Mode examples:
+
+> [SeleniumBase/examples/cdp_mode](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode)
+
+### π Example 1: (Pokemon site using Incapsula/Imperva protection with invisible reCAPTCHA)
+
+> [SeleniumBase/examples/cdp_mode/raw_pokemon.py](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode/raw_pokemon.py)
+
+
+
+ βΆοΈ (Click to expand code preview )
+
+```python
+from seleniumbase import SB
+
+with SB(uc=True, test=True, locale_code="en") as sb:
+ url = "https://www.pokemon.com/us"
+ sb.activate_cdp_mode(url)
+ sb.sleep(1)
+ sb.cdp.click_if_visible("button#onetrust-reject-all-handler")
+ sb.cdp.click('a[href="https://www.pokemon.com/us/pokedex/"]')
+ sb.sleep(1)
+ sb.cdp.click('b:contains("Show Advanced Search")')
+ sb.sleep(1)
+ sb.cdp.click('span[data-type="type"][data-value="electric"]')
+ sb.cdp.click("a#advSearch")
+ sb.sleep(1)
+ sb.cdp.click('img[src*="img/pokedex/detail/025.png"]')
+ sb.cdp.assert_text("Pikachu", 'div[class*="title"]')
+ sb.cdp.assert_element('img[alt="Pikachu"]')
+ sb.cdp.scroll_into_view("div.pokemon-ability-info")
+ sb.sleep(1)
+ sb.cdp.flash('div[class*="title"]')
+ sb.cdp.flash('img[alt="Pikachu"]')
+ sb.cdp.flash("div.pokemon-ability-info")
+ name = sb.cdp.get_text("label.styled-select")
+ info = sb.cdp.get_text("div.version-descriptions p.active")
+ print("*** %s: ***\n* %s" % (name, info))
+ sb.sleep(2)
+ sb.cdp.highlight_overlay("div.pokemon-ability-info")
+ sb.sleep(2)
+ sb.cdp.click('a[href="https://www.pokemon.com/us/play-pokemon/"]')
+ sb.cdp.click('h3:contains("Find an Event")')
+ location = "Concord, MA, USA"
+ sb.cdp.type('input[data-testid="location-search"]', location)
+ sb.sleep(1)
+ sb.cdp.click("div.autocomplete-dropdown-container div.suggestion-item")
+ sb.cdp.click('img[alt="search-icon"]')
+ sb.sleep(2)
+ events = sb.cdp.select_all('div[data-testid="event-name"]')
+ print("*** Pokemon events near %s: ***" % location)
+ for event in events:
+ print("* " + event.text)
+ sb.sleep(2)
+```
+
+
+
+### π Example 2: (Hyatt site using Kasada protection)
+
+> [SeleniumBase/examples/cdp_mode/raw_hyatt.py](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode/raw_hyatt.py)
+
+
+
+ βΆοΈ (Click to expand code preview )
+
+```python
+from seleniumbase import SB
+
+with SB(uc=True, test=True, locale_code="en") as sb:
+ url = "https://www.hyatt.com/"
+ sb.activate_cdp_mode(url)
+ sb.sleep(1)
+ sb.cdp.click_if_visible('button[aria-label="Close"]')
+ sb.sleep(0.5)
+ sb.cdp.click('span:contains("Explore")')
+ sb.sleep(1)
+ sb.cdp.click('a:contains("Hotels & Resorts")')
+ sb.sleep(2.5)
+ location = "Anaheim, CA, USA"
+ sb.cdp.press_keys("input#searchbox", location)
+ sb.sleep(1)
+ sb.cdp.click("div#suggestion-list ul li a")
+ sb.sleep(1)
+ sb.cdp.click('div.hotel-card-footer button')
+ sb.sleep(1)
+ sb.cdp.click('button[data-locator="find-hotels"]')
+ sb.sleep(4)
+ hotel_names = sb.cdp.select_all(
+ 'div[data-booking-status="BOOKABLE"] [class*="HotelCard_header"]'
+ )
+ hotel_prices = sb.cdp.select_all(
+ 'div[data-booking-status="BOOKABLE"] div.rate-currency'
+ )
+ sb.assert_true(len(hotel_names) == len(hotel_prices))
+ print("Hyatt Hotels in %s:" % location)
+ print("(" + sb.cdp.get_text("ul.b-color_text-white") + ")")
+ if len(hotel_names) == 0:
+ print("No availability over the selected dates!")
+ for i, hotel in enumerate(hotel_names):
+ print("* %s: %s => %s" % (i + 1, hotel.text, hotel_prices[i].text))
+```
+
+
+
+### π Example 3: (BestWestern site using DataDome protection)
+
+* [SeleniumBase/examples/cdp_mode/raw_bestwestern.py](https://github.com/seleniumbase/SeleniumBase/tree/master/examples/cdp_mode/raw_bestwestern.py)
+
+
+
+ βΆοΈ (Click to expand code preview )
+
+```python
+from seleniumbase import SB
+
+with SB(uc=True, test=True, locale_code="en") as sb:
+ url = "https://www.bestwestern.com/en_US.html"
+ sb.activate_cdp_mode(url)
+ sb.sleep(1.5)
+ sb.cdp.click_if_visible("div.onetrust-close-btn-handler")
+ sb.sleep(0.5)
+ sb.cdp.click("input#destination-input")
+ sb.sleep(1.5)
+ location = "Palm Springs, CA, USA"
+ sb.cdp.press_keys("input#destination-input", location)
+ sb.sleep(0.6)
+ sb.cdp.click("ul#google-suggestions li")
+ sb.sleep(0.6)
+ sb.cdp.click("button#btn-modify-stay-update")
+ sb.sleep(1.5)
+ sb.cdp.click("label#available-label")
+ sb.sleep(4)
+ print("Best Western Hotels in %s:" % location)
+ summary_details = sb.cdp.get_text("#summary-details-column")
+ dates = summary_details.split("ROOM")[0].split("DATES")[-1].strip()
+ print("(Dates: %s)" % dates)
+ flip_cards = sb.cdp.select_all(".flipCard")
+ for i, flip_card in enumerate(flip_cards):
+ hotel = flip_card.query_selector(".hotelName")
+ price = flip_card.query_selector(".priceSection")
+ if hotel and price:
+ print("* %s: %s => %s" % (
+ i + 1, hotel.text.strip(), price.text.strip())
+ )
+```
+
+
+
+(Note: Extra sb.sleep() calls have been added to prevent bot-detection because some sites will flag you as a bot if you perform actions too quickly.)
+
+(Note: Some sites may IP-block you for 36 hours or more if they catch you using regular Selenium WebDriver . Be extra careful when creating and/or modifying automation scripts that run on them.)
+
+--------
+
+### π CDP Mode API / Methods
+
+(Some method args have been left out for simplicity. Eg: timeout)
+
+```python
+sb.cdp.get(url)
+sb.cdp.reload()
+sb.cdp.refresh()
+sb.cdp.add_handler(event, handler)
+sb.cdp.find_element(selector)
+sb.cdp.find_all(selector)
+sb.cdp.find_elements_by_text(text, tag_name=None)
+sb.cdp.select(selector)
+sb.cdp.select_all(selector)
+sb.cdp.click_link(link_text)
+sb.cdp.tile_windows(windows=None, max_columns=0)
+sb.cdp.get_all_cookies(*args, **kwargs)
+sb.cdp.set_all_cookies(*args, **kwargs)
+sb.cdp.save_cookies(*args, **kwargs)
+sb.cdp.load_cookies(*args, **kwargs)
+sb.cdp.clear_cookies(*args, **kwargs)
+sb.cdp.sleep(seconds)
+sb.cdp.bring_active_window_to_front()
+sb.cdp.get_active_element()
+sb.cdp.get_active_element_css()
+sb.cdp.click(selector)
+sb.cdp.click_active_element()
+sb.cdp.click_if_visible(selector)
+sb.cdp.mouse_click(selector)
+sb.cdp.nested_click(parent_selector, selector)
+sb.cdp.get_nested_element(parent_selector, selector)
+sb.cdp.flash(selector)
+sb.cdp.focus(selector)
+sb.cdp.highlight_overlay(selector)
+sb.cdp.remove_element(selector)
+sb.cdp.remove_from_dom(selector)
+sb.cdp.remove_elements(selector)
+sb.cdp.scroll_into_view(selector)
+sb.cdp.send_keys(selector, text)
+sb.cdp.press_keys(selector, text)
+sb.cdp.type(selector, text)
+sb.cdp.evaluate(expression)
+sb.cdp.js_dumps(obj_name)
+sb.cdp.maximize()
+sb.cdp.minimize()
+sb.cdp.medimize()
+sb.cdp.set_window_rect()
+sb.cdp.reset_window_size()
+sb.cdp.get_window()
+sb.cdp.get_text()
+sb.cdp.get_title()
+sb.cdp.get_current_url()
+sb.cdp.get_origin()
+sb.cdp.get_page_source()
+sb.cdp.get_user_agent()
+sb.cdp.get_cookie_string()
+sb.cdp.get_locale_code()
+sb.cdp.get_screen_rect()
+sb.cdp.get_window_rect()
+sb.cdp.get_window_size()
+sb.cdp.get_window_position()
+sb.cdp.get_element_rect(selector)
+sb.cdp.get_element_size(selector)
+sb.cdp.get_element_position(selector)
+sb.cdp.get_gui_element_rect(selector)
+sb.cdp.get_gui_element_center(selector)
+sb.cdp.get_document()
+sb.cdp.get_flattened_document()
+sb.cdp.get_element_attributes(selector)
+sb.cdp.get_element_html(selector)
+sb.cdp.set_attributes(selector, attribute, value)
+sb.cdp.internalize_links()
+sb.cdp.is_element_present(selector)
+sb.cdp.is_element_visible(selector)
+sb.cdp.assert_element(selector)
+sb.cdp.assert_element_present(selector)
+sb.cdp.assert_text(text, selector="html")
+sb.cdp.assert_exact_text(text, selector="html")
+sb.cdp.save_screenshot(name, folder=None, selector=None)
+```
+
+--------
+
+### π CDP Mode WebElement API / Methods
+
+```python
+element.clear_input()
+element.click()
+element.flash()
+element.focus()
+element.highlight_overlay()
+element.mouse_click()
+element.mouse_drag(destination)
+element.mouse_move()
+element.query_selector(selector)
+element.querySelector(selector)
+element.query_selector_all(selector)
+element.querySelectorAll(selector)
+element.remove_from_dom()
+element.save_screenshot(*args, **kwargs)
+element.save_to_dom()
+element.scroll_into_view()
+element.select_option()
+element.send_file(*file_paths)
+element.send_keys(text)
+element.set_text(value)
+element.type(text)
+element.get_position()
+element.get_html()
+element.get_js_attributes()
+```
+
+--------
+
+π Methods / APIs |
π Tours
π‘ Syntax Formats |
+π Syntax Formats |
π€ CI/CD
β»οΈ Boilerplates |
@@ -40,6 +40,8 @@
ποΈ GUI |
π€ UC Mode
+π CDP Mode
--------
diff --git a/help_docs/customizing_test_runs.md b/help_docs/customizing_test_runs.md
index 3caced9ae90..4eb991d641a 100644
--- a/help_docs/customizing_test_runs.md
+++ b/help_docs/customizing_test_runs.md
@@ -2,7 +2,7 @@
## [
diff --git a/help_docs/method_summary.md b/help_docs/method_summary.md
index fdce9beeecc..d79ba678b36 100644
--- a/help_docs/method_summary.md
+++ b/help_docs/method_summary.md
@@ -125,6 +125,8 @@ self.remove_attribute(selector, attribute, by="css selector", timeout=None)
self.remove_attributes(selector, attribute, by="css selector")
+self.internalize_links()
+
self.get_property(selector, property, by="css selector", timeout=None)
self.get_text_content(selector="html", by="css selector", timeout=None)
@@ -134,6 +136,8 @@ self.get_property_value(selector, property, by="css selector", timeout=None)
self.get_image_url(selector, by="css selector", timeout=None)
self.find_elements(selector, by="css selector", limit=0)
+# Duplicates:
+# self.select_all(selector, by="css selector", limit=0)
self.find_visible_elements(selector, by="css selector", limit=0)
diff --git a/help_docs/recorder_mode.md b/help_docs/recorder_mode.md
index e859097a151..cf6a78b5a24 100644
--- a/help_docs/recorder_mode.md
+++ b/help_docs/recorder_mode.md
@@ -59,7 +59,7 @@ sbase recorder
βΊοΈ While a recording is in progress, you can press the ``[ESC]`` key to pause the Recorder. To resume the recording, you can hit the ``[~`]`` key, which is located directly below the ``[ESC]`` key on most keyboards.
-βΊοΈ From within Recorder Mode there are two additional modes: "Assert Element Mode" and "Assert Text Mode". To switch into "Assert Element Mode", press the ``[^]-key (SHIFT+6)``: The border will become purple, and you'll be able to click on elements to assert from your test. To switch into "Assert Text Mode", press the ``[&]-key (SHIFT+7)``: The border will become teal, and you'll be able to click on elements for asserting text from your test.
+βΊοΈ From within Recorder Mode there are two additional modes: "Assert Element Mode" and "Assert Text Mode". To switch into "Assert Element Mode", press the ``[^]-key (SHIFT+6 on standard QWERTY keyboards)``: The border will become purple, and you'll be able to click on elements to assert from your test. To switch into "Assert Text Mode", press the ``[&]-key (SHIFT+7 on standard QWERTY keyboards)``: The border will become teal, and you'll be able to click on elements for asserting text from your test.
βΊοΈ While using either of the two special Assertion Modes, certain actions such as clicking on links won't have any effect. This lets you make assertions on elements without navigating away from the page, etc. To add an assertion for buttons without triggering default "click" behavior, mouse-down on the button and then mouse-up somewhere else. (This prevents a detected click while still recording the assert.) To return back to the original Recorder Mode, press any key other than ``[SHIFT]`` or ``[BACKSPACE]`` (Eg: Press ``[CONTROL]``, etc.). Press ``[ESC]`` once to leave the Assertion Modes, but it'll stop the Recorder if you press it again.
@@ -123,6 +123,8 @@ pytest TEST_NAME.py --trace --rec -s
βΊοΈ By launching the Recorder App with sbase recorder --ee, you can end the recording by pressing {SHIFT+ESC} instead of the usual way of ending the recording by typing c from a breakpoint() and pressing Enter. Those buttons don't need to be pressed at the same time, but SHIFT must be pressed directly before ESC.
+βΊοΈ Use sbase recorder --uc to launch the Recorder App with UC Mode enabled. (The driver will be disconnected from Chrome, but the Recorder extension will still capture any browser actions.)
+
--------
To learn more about SeleniumBase, check out the Docs Site:
diff --git a/help_docs/syntax_formats.md b/help_docs/syntax_formats.md
index d96e72d9bf7..eff59a301a0 100644
--- a/help_docs/syntax_formats.md
+++ b/help_docs/syntax_formats.md
@@ -4,7 +4,7 @@
π‘ SeleniumBase supports multiple ways of structuring tests:
+π SeleniumBase supports multiple ways of structuring tests:
diff --git a/mkdocs.yml b/mkdocs.yml
index ea530a02353..dc02424dad9 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -115,6 +115,7 @@ nav:
- ποΈ Presentation Maker: examples/presenter/ReadMe.md
- Integrations:
- π€ UC Mode: help_docs/uc_mode.md
+ - π CDP Mode: examples/cdp_mode/ReadMe.md
- π€ GitHub CI: integrations/github/workflows/ReadMe.md
- π MasterQA: seleniumbase/masterqa/ReadMe.md
- ποΈ Case Plans: help_docs/case_plans.md
diff --git a/mkdocs_build/requirements.txt b/mkdocs_build/requirements.txt
index e82d61fc56c..3260df8b31b 100644
--- a/mkdocs_build/requirements.txt
+++ b/mkdocs_build/requirements.txt
@@ -6,8 +6,8 @@ pymdown-extensions>=10.11.2
pipdeptree>=2.23.4
python-dateutil>=2.8.2
Markdown==3.7
-markdown2==2.5.0
-MarkupSafe==2.1.5
+markdown2==2.5.1
+MarkupSafe==3.0.2
Jinja2==3.1.4
click==8.1.7
ghp-import==2.1.0
@@ -20,7 +20,7 @@ lxml==5.3.0
pyquery==2.0.1
readtime==3.0.0
mkdocs==1.6.1
-mkdocs-material==9.5.39
+mkdocs-material==9.5.42
mkdocs-exclude-search==0.6.6
mkdocs-simple-hooks==0.1.5
mkdocs-material-extensions==1.3.1
diff --git a/pyproject.toml b/pyproject.toml
index 2697d39bd76..328dd55b152 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -13,6 +13,7 @@ dynamic = [
"keywords",
"classifiers",
"description",
+ "maintainers",
"entry-points",
"dependencies",
"requires-python",
diff --git a/requirements.txt b/requirements.txt
index 37d58a08012..286c50fd313 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,13 +1,15 @@
-pip>=24.1.2
+pip>=24.2
packaging>=24.1
setuptools~=70.2;python_version<"3.10"
-setuptools>=70.2.0;python_version>="3.10"
+setuptools>=73.0.1;python_version>="3.10"
wheel>=0.44.0
attrs>=24.2.0
certifi>=2024.8.30
exceptiongroup>=1.2.2
+websockets>=13.1
filelock>=3.16.1
fasteners>=0.19
+mycdp>=1.0.1
pynose>=1.5.3
platformdirs>=4.3.6
typing-extensions>=4.12.2
@@ -23,14 +25,14 @@ tabcompleter>=1.3.3
pdbp>=1.5.4
idna==3.10
chardet==5.2.0
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
urllib3>=1.26.20,<2;python_version<"3.10"
urllib3>=1.26.20,<2.3.0;python_version>="3.10"
-requests==2.31.0
+requests==2.32.3
sniffio==1.3.1
h11==0.14.0
outcome==1.3.0.post0
-trio==0.26.2
+trio==0.27.0
trio-websocket==0.11.1
wsproto==1.2.0
websocket-client==1.8.0
@@ -55,12 +57,13 @@ pyotp==2.9.0
python-xlib==0.33;platform_system=="Linux"
markdown-it-py==3.0.0
mdurl==0.1.2
-rich==13.9.2
+rich==13.9.3
# --- Testing Requirements --- #
# ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.)
-coverage>=7.6.1
+coverage>=7.6.1;python_version<"3.9"
+coverage>=7.6.4;python_version>="3.9"
pytest-cov>=5.0.0
flake8==5.0.4;python_version<"3.9"
flake8==7.1.1;python_version>="3.9"
diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py
index 4104caa2cc4..73cd0a1e3e6 100755
--- a/seleniumbase/__version__.py
+++ b/seleniumbase/__version__.py
@@ -1,2 +1,2 @@
# seleniumbase package
-__version__ = "4.31.6"
+__version__ = "4.32.0"
diff --git a/seleniumbase/common/decorators.py b/seleniumbase/common/decorators.py
index 6c5a3221c69..0a1850716cc 100644
--- a/seleniumbase/common/decorators.py
+++ b/seleniumbase/common/decorators.py
@@ -43,23 +43,24 @@ def my_method():
finally:
end_time = time.time()
run_time = end_time - start_time
+ name = description
# Print times with a statistically significant number of decimal places
if run_time < 0.0001:
- print(" {%s} ran for %.7f seconds." % (description, run_time))
+ print(" - {%s} ran for %.7f seconds." % (name, run_time))
elif run_time < 0.001:
- print(" {%s} ran for %.6f seconds." % (description, run_time))
+ print(" - {%s} ran for %.6f seconds." % (name, run_time))
elif run_time < 0.01:
- print(" {%s} ran for %.5f seconds." % (description, run_time))
+ print(" - {%s} ran for %.5f seconds." % (name, run_time))
elif run_time < 0.1:
- print(" {%s} ran for %.4f seconds." % (description, run_time))
+ print(" - {%s} ran for %.4f seconds." % (name, run_time))
elif run_time < 1:
- print(" {%s} ran for %.3f seconds." % (description, run_time))
+ print(" - {%s} ran for %.3f seconds." % (name, run_time))
else:
- print(" {%s} ran for %.2f seconds." % (description, run_time))
+ print(" - {%s} ran for %.2f seconds." % (name, run_time))
if limit and limit > 0 and run_time > limit:
message = (
"\n {%s} duration of %.2fs exceeded the time limit of %.2fs!"
- % (description, run_time, limit)
+ % (name, run_time, limit)
)
if exception:
message = exception.msg + "\nAND " + message
diff --git a/seleniumbase/console_scripts/sb_install.py b/seleniumbase/console_scripts/sb_install.py
index cc529097898..8d55ac87bb1 100644
--- a/seleniumbase/console_scripts/sb_install.py
+++ b/seleniumbase/console_scripts/sb_install.py
@@ -811,7 +811,7 @@ def main(override=None, intel_for_uc=None, force_uc=None):
zip_ref.extractall(downloads_folder)
zip_ref.close()
os.remove(zip_file_path)
- shutil.copyfile(driver_path, os.path.join(downloads_folder, filename))
+ shutil.copy3(driver_path, os.path.join(downloads_folder, filename))
log_d("%sUnzip Complete!%s\n" % (c2, cr))
to_remove = [
"%s/%s/ruby_example/Gemfile" % (downloads_folder, h_ie_fn),
@@ -953,7 +953,7 @@ def main(override=None, intel_for_uc=None, force_uc=None):
)
if copy_to_path and os.path.exists(LOCAL_PATH):
path_file = LOCAL_PATH + f_name
- shutil.copyfile(new_file, path_file)
+ shutil.copy2(new_file, path_file)
make_executable(path_file)
log_d("Also copied to: %s%s%s" % (c3, path_file, cr))
log_d("")
@@ -1042,7 +1042,7 @@ def main(override=None, intel_for_uc=None, force_uc=None):
)
if copy_to_path and os.path.exists(LOCAL_PATH):
path_file = LOCAL_PATH + f_name
- shutil.copyfile(new_file, path_file)
+ shutil.copy2(new_file, path_file)
make_executable(path_file)
log_d("Also copied to: %s%s%s" % (c3, path_file, cr))
log_d("")
@@ -1078,7 +1078,7 @@ def main(override=None, intel_for_uc=None, force_uc=None):
)
if copy_to_path and os.path.exists(LOCAL_PATH):
path_file = LOCAL_PATH + f_name
- shutil.copyfile(new_file, path_file)
+ shutil.copy2(new_file, path_file)
make_executable(path_file)
log_d("Also copied to: %s%s%s" % (c3, path_file, cr))
log_d("")
diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py
index 01cba62c226..f92a0a7acf4 100644
--- a/seleniumbase/core/browser_launcher.py
+++ b/seleniumbase/core/browser_launcher.py
@@ -29,6 +29,7 @@
from seleniumbase.core import download_helper
from seleniumbase.core import proxy_helper
from seleniumbase.core import sb_driver
+from seleniumbase.core import sb_cdp
from seleniumbase.fixtures import constants
from seleniumbase.fixtures import js_utils
from seleniumbase.fixtures import page_actions
@@ -153,8 +154,10 @@ def extend_driver(driver):
page.find_element = DM.find_element
page.find_elements = DM.find_elements
page.locator = DM.locator
+ page.get_current_url = DM.get_current_url
page.get_page_source = DM.get_page_source
page.get_title = DM.get_title
+ page.get_page_title = DM.get_title
page.switch_to_default_window = DM.switch_to_default_window
page.switch_to_newest_window = DM.switch_to_newest_window
page.open_new_window = DM.open_new_window
@@ -206,6 +209,9 @@ def extend_driver(driver):
driver.is_valid_url = DM.is_valid_url
driver.is_alert_present = DM.is_alert_present
driver.is_online = DM.is_online
+ driver.is_connected = DM.is_connected
+ driver.is_uc_mode_active = DM.is_uc_mode_active
+ driver.is_cdp_mode_active = DM.is_cdp_mode_active
driver.js_click = DM.js_click
driver.get_text = DM.get_text
driver.get_active_element_css = DM.get_active_element_css
@@ -217,8 +223,10 @@ def extend_driver(driver):
driver.highlight_if_visible = DM.highlight_if_visible
driver.sleep = time.sleep
driver.get_attribute = DM.get_attribute
+ driver.get_current_url = DM.get_current_url
driver.get_page_source = DM.get_page_source
driver.get_title = DM.get_title
+ driver.get_page_title = DM.get_title
driver.switch_to_default_window = DM.switch_to_default_window
driver.switch_to_newest_window = DM.switch_to_newest_window
driver.open_new_window = DM.open_new_window
@@ -362,6 +370,11 @@ def has_captcha(text):
return False
+def __is_cdp_swap_needed(driver):
+ """If the driver is disconnected, use a CDP method when available."""
+ return shared_utils.is_cdp_swap_needed(driver)
+
+
def uc_special_open_if_cf(
driver,
url,
@@ -432,10 +445,11 @@ def uc_special_open_if_cf(
def uc_open(driver, url):
- if url.startswith("//"):
- url = "https:" + url
- elif ":" not in url:
- url = "https://" + url
+ url = shared_utils.fix_url_as_needed(url)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.get(url)
+ time.sleep(0.3)
+ return
if (url.startswith("http:") or url.startswith("https:")):
with driver:
script = 'window.location.href = "%s";' % url
@@ -446,10 +460,11 @@ def uc_open(driver, url):
def uc_open_with_tab(driver, url):
- if url.startswith("//"):
- url = "https:" + url
- elif ":" not in url:
- url = "https://" + url
+ url = shared_utils.fix_url_as_needed(url)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.get(url)
+ time.sleep(0.3)
+ return
if (url.startswith("http:") or url.startswith("https:")):
with driver:
driver.execute_script('window.open("%s","_blank");' % url)
@@ -462,12 +477,13 @@ def uc_open_with_tab(driver, url):
def uc_open_with_reconnect(driver, url, reconnect_time=None):
"""Open a url, disconnect chromedriver, wait, and reconnect."""
+ url = shared_utils.fix_url_as_needed(url)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.get(url)
+ time.sleep(0.3)
+ return
if not reconnect_time:
reconnect_time = constants.UC.RECONNECT_TIME
- if url.startswith("//"):
- url = "https:" + url
- elif ":" not in url:
- url = "https://" + url
if (url.startswith("http:") or url.startswith("https:")):
script = 'window.open("%s","_blank");' % url
driver.execute_script(script)
@@ -493,15 +509,157 @@ def uc_open_with_reconnect(driver, url, reconnect_time=None):
return None
+def uc_open_with_cdp_mode(driver, url=None):
+ import asyncio
+ from seleniumbase.undetected.cdp_driver import cdp_util
+
+ current_url = None
+ try:
+ current_url = driver.current_url
+ except Exception:
+ driver.connect()
+ current_url = driver.current_url
+ url_protocol = current_url.split(":")[0]
+ if url_protocol not in ["about", "data", "chrome"]:
+ script = 'window.open("data:,","_blank");'
+ js_utils.call_me_later(driver, script, 3)
+ time.sleep(0.012)
+ driver.close()
+ driver.clear_cdp_listeners()
+ driver.delete_all_cookies()
+ driver.delete_network_conditions()
+ driver.disconnect()
+
+ cdp_details = driver._get_cdp_details()
+ cdp_host = cdp_details[1].split("://")[1].split(":")[0]
+ cdp_port = int(cdp_details[1].split("://")[1].split(":")[1].split("/")[0])
+
+ url = shared_utils.fix_url_as_needed(url)
+ url_protocol = url.split(":")[0]
+ safe_url = True
+ if url_protocol not in ["about", "data", "chrome"]:
+ safe_url = False
+
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ driver.cdp_base = loop.run_until_complete(
+ cdp_util.start(host=cdp_host, port=cdp_port)
+ )
+ page = loop.run_until_complete(driver.cdp_base.get(url))
+ if not safe_url:
+ time.sleep(constants.UC.CDP_MODE_OPEN_WAIT)
+ cdp = types.SimpleNamespace()
+ CDPM = sb_cdp.CDPMethods(loop, page, driver)
+ cdp.get = CDPM.get
+ cdp.open = CDPM.get
+ cdp.reload = CDPM.reload
+ cdp.refresh = CDPM.refresh
+ cdp.add_handler = CDPM.add_handler
+ cdp.get_event_loop = CDPM.get_event_loop
+ cdp.find_element = CDPM.find_element
+ cdp.find = CDPM.find_element
+ cdp.locator = CDPM.find_element
+ cdp.find_all = CDPM.find_all
+ cdp.find_elements_by_text = CDPM.find_elements_by_text
+ cdp.select = CDPM.select
+ cdp.select_all = CDPM.select_all
+ cdp.click_link = CDPM.click_link
+ cdp.tile_windows = CDPM.tile_windows
+ cdp.get_all_cookies = CDPM.get_all_cookies
+ cdp.set_all_cookies = CDPM.set_all_cookies
+ cdp.save_cookies = CDPM.save_cookies
+ cdp.load_cookies = CDPM.load_cookies
+ cdp.clear_cookies = CDPM.clear_cookies
+ cdp.bring_active_window_to_front = CDPM.bring_active_window_to_front
+ cdp.bring_to_front = CDPM.bring_active_window_to_front
+ cdp.get_active_element = CDPM.get_active_element
+ cdp.get_active_element_css = CDPM.get_active_element_css
+ cdp.click = CDPM.click
+ cdp.click_active_element = CDPM.click_active_element
+ cdp.click_if_visible = CDPM.click_if_visible
+ cdp.mouse_click = CDPM.mouse_click
+ cdp.remove_element = CDPM.remove_element
+ cdp.remove_from_dom = CDPM.remove_from_dom
+ cdp.remove_elements = CDPM.remove_elements
+ cdp.scroll_into_view = CDPM.scroll_into_view
+ cdp.send_keys = CDPM.send_keys
+ cdp.press_keys = CDPM.press_keys
+ cdp.type = CDPM.type
+ cdp.evaluate = CDPM.evaluate
+ cdp.js_dumps = CDPM.js_dumps
+ cdp.maximize = CDPM.maximize
+ cdp.minimize = CDPM.minimize
+ cdp.medimize = CDPM.medimize
+ cdp.set_window_rect = CDPM.set_window_rect
+ cdp.reset_window_size = CDPM.reset_window_size
+ cdp.set_attributes = CDPM.set_attributes
+ cdp.internalize_links = CDPM.internalize_links
+ cdp.get_window = CDPM.get_window
+ cdp.get_element_attributes = CDPM.get_element_attributes
+ cdp.get_element_html = CDPM.get_element_html
+ cdp.get_element_rect = CDPM.get_element_rect
+ cdp.get_element_size = CDPM.get_element_size
+ cdp.get_element_position = CDPM.get_element_position
+ cdp.get_gui_element_rect = CDPM.get_gui_element_rect
+ cdp.get_gui_element_center = CDPM.get_gui_element_center
+ cdp.get_page_source = CDPM.get_page_source
+ cdp.get_user_agent = CDPM.get_user_agent
+ cdp.get_cookie_string = CDPM.get_cookie_string
+ cdp.get_locale_code = CDPM.get_locale_code
+ cdp.get_text = CDPM.get_text
+ cdp.get_title = CDPM.get_title
+ cdp.get_page_title = CDPM.get_title
+ cdp.get_current_url = CDPM.get_current_url
+ cdp.get_origin = CDPM.get_origin
+ cdp.get_nested_element = CDPM.get_nested_element
+ cdp.get_document = CDPM.get_document
+ cdp.get_flattened_document = CDPM.get_flattened_document
+ cdp.get_screen_rect = CDPM.get_screen_rect
+ cdp.get_window_rect = CDPM.get_window_rect
+ cdp.get_window_size = CDPM.get_window_size
+ cdp.nested_click = CDPM.nested_click
+ cdp.flash = CDPM.flash
+ cdp.focus = CDPM.focus
+ cdp.highlight_overlay = CDPM.highlight_overlay
+ cdp.get_window_position = CDPM.get_window_position
+ cdp.is_element_present = CDPM.is_element_present
+ cdp.is_element_visible = CDPM.is_element_visible
+ cdp.assert_element_present = CDPM.assert_element_present
+ cdp.assert_element = CDPM.assert_element
+ cdp.assert_element_visible = CDPM.assert_element
+ cdp.assert_text = CDPM.assert_text
+ cdp.assert_exact_text = CDPM.assert_exact_text
+ cdp.save_screenshot = CDPM.save_screenshot
+ cdp.page = page # async world
+ cdp.driver = driver.cdp_base # async world
+ cdp.tab = cdp.page # shortcut (original)
+ cdp.browser = driver.cdp_base # shortcut (original)
+ cdp.util = cdp_util # shortcut (original)
+ core_items = types.SimpleNamespace()
+ core_items.browser = cdp.browser
+ core_items.tab = cdp.tab
+ core_items.util = cdp.util
+ cdp.core = core_items
+ driver.cdp = cdp
+ driver._is_using_cdp = True
+
+
+def uc_activate_cdp_mode(driver, url=None):
+ uc_open_with_cdp_mode(driver, url=url)
+
+
def uc_open_with_disconnect(driver, url, timeout=None):
"""Open a url and disconnect chromedriver.
Then waits for the duration of the timeout.
Note: You can't perform Selenium actions again
until after you've called driver.connect()."""
- if url.startswith("//"):
- url = "https:" + url
- elif ":" not in url:
- url = "https://" + url
+ url = shared_utils.fix_url_as_needed(url)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.get(url)
+ time.sleep(0.3)
+ return
+ if not driver.is_connected():
+ driver.connect()
if (url.startswith("http:") or url.startswith("https:")):
script = 'window.open("%s","_blank");' % url
driver.execute_script(script)
@@ -685,6 +843,9 @@ def uc_gui_write(driver, text):
def get_gui_element_position(driver, selector):
+ if __is_cdp_swap_needed(driver):
+ element_rect = driver.cdp.get_gui_element_rect(selector)
+ return (element_rect["x"], element_rect["y"])
element = driver.wait_for_element_present(selector, timeout=3)
element_rect = element.rect
window_rect = driver.get_window_rect()
@@ -823,6 +984,7 @@ def _uc_gui_click_captcha(
blind=False,
ctype=None,
):
+ cdp_mode_on_at_start = __is_cdp_swap_needed(driver)
_on_a_captcha_page = None
if ctype == "cf_t":
if not _on_a_cf_turnstile_page(driver):
@@ -864,10 +1026,13 @@ def _uc_gui_click_captcha(
is_in_frame = js_utils.is_in_frame(driver)
if not is_in_frame:
# Make sure the window is on top
- page_actions.switch_to_window(
- driver, driver.current_window_handle, 2, uc_lock=False
- )
- if IS_WINDOWS:
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.bring_active_window_to_front()
+ else:
+ page_actions.switch_to_window(
+ driver, driver.current_window_handle, 2, uc_lock=False
+ )
+ if IS_WINDOWS and not __is_cdp_swap_needed(driver):
window_rect = driver.get_window_rect()
width = window_rect["width"]
height = window_rect["height"]
@@ -950,7 +1115,10 @@ def _uc_gui_click_captcha(
new_class = the_class.replaceAll('center', 'left');
$elements[index].setAttribute('class', new_class);}"""
)
- driver.execute_script(script)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.evaluate(script)
+ else:
+ driver.execute_script(script)
if not is_in_frame or needs_switch:
# Currently not in frame (or nested frame outside CF one)
try:
@@ -961,16 +1129,22 @@ def _uc_gui_click_captcha(
if visible_iframe:
if driver.is_element_present("iframe"):
i_x, i_y = get_gui_element_position(driver, "iframe")
- driver.switch_to_frame("iframe")
+ if driver.is_connected():
+ driver.switch_to_frame("iframe")
else:
return
if not i_x or not i_y:
return
try:
- if visible_iframe:
+ if ctype == "g_rc" and not driver.is_connected():
+ x = (i_x + 32) * width_ratio
+ y = (i_y + 34) * width_ratio
+ elif visible_iframe:
selector = "span"
if ctype == "g_rc":
selector = "span.recaptcha-checkbox"
+ if not driver.is_connected():
+ selector = "iframe"
element = driver.wait_for_element_present(
selector, timeout=2.5
)
@@ -981,16 +1155,18 @@ def _uc_gui_click_captcha(
else:
x = (i_x + 34) * width_ratio
y = (i_y + 34) * width_ratio
- driver.switch_to.default_content()
- except Exception:
- try:
+ if driver.is_connected():
driver.switch_to.default_content()
- except Exception:
- return
+ except Exception:
+ if driver.is_connected():
+ try:
+ driver.switch_to.default_content()
+ except Exception:
+ return
if x and y:
sb_config._saved_cf_x_y = (x, y)
if driver.is_element_present(".footer .clearfix .ray-id"):
- driver.uc_open_with_disconnect(driver.current_url, 3.8)
+ driver.uc_open_with_disconnect(driver.get_current_url(), 3.8)
else:
driver.disconnect()
with suppress(Exception):
@@ -1013,9 +1189,12 @@ def _uc_gui_click_captcha(
if retry and x and y and (caught or _on_a_captcha_page(driver)):
with gui_lock: # Prevent issues with multiple processes
# Make sure the window is on top
- page_actions.switch_to_window(
- driver, driver.current_window_handle, 2, uc_lock=False
- )
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.bring_active_window_to_front()
+ else:
+ page_actions.switch_to_window(
+ driver, driver.current_window_handle, 2, uc_lock=False
+ )
if driver.is_element_present("iframe"):
try:
driver.switch_to_frame(frame)
@@ -1035,14 +1214,18 @@ def _uc_gui_click_captcha(
driver.switch_to.parent_frame(checkbox_success)
return
if blind:
- driver.uc_open_with_disconnect(driver.current_url, 3.8)
- _uc_gui_click_x_y(driver, x, y, timeframe=0.32)
+ driver.uc_open_with_disconnect(driver.get_current_url(), 3.8)
+ if __is_cdp_swap_needed(driver) and _on_a_captcha_page(driver):
+ _uc_gui_click_x_y(driver, x, y, timeframe=0.32)
+ else:
+ time.sleep(0.1)
else:
- driver.uc_open_with_reconnect(driver.current_url, 3.8)
+ driver.uc_open_with_reconnect(driver.get_current_url(), 3.8)
if _on_a_captcha_page(driver):
driver.disconnect()
_uc_gui_click_x_y(driver, x, y, timeframe=0.32)
- driver.reconnect(reconnect_time)
+ if not cdp_mode_on_at_start:
+ driver.reconnect(reconnect_time)
def uc_gui_click_captcha(driver, frame="iframe", retry=False, blind=False):
@@ -1089,6 +1272,9 @@ def _uc_gui_handle_captcha_(driver, frame="iframe", ctype=None):
ctype = "cf_t"
else:
return
+ if not driver.is_connected():
+ driver.connect()
+ time.sleep(2)
install_pyautogui_if_missing(driver)
import pyautogui
pyautogui = get_configured_pyautogui(pyautogui)
@@ -1111,9 +1297,12 @@ def _uc_gui_handle_captcha_(driver, frame="iframe", ctype=None):
page_actions.switch_to_window(
driver, driver.current_window_handle, 2, uc_lock=False
)
- if IS_WINDOWS and hasattr(pyautogui, "getActiveWindowTitle"):
+ if (
+ IS_WINDOWS
+ and hasattr(pyautogui, "getActiveWindowTitle")
+ ):
py_a_g_title = pyautogui.getActiveWindowTitle()
- window_title = driver.title
+ window_title = driver.get_title()
if not py_a_g_title.startswith(window_title):
window_rect = driver.get_window_rect()
width = window_rect["width"]
@@ -1614,6 +1803,7 @@ def _set_chrome_options(
prefs["default_content_settings.popups"] = 0
prefs["managed_default_content_settings.popups"] = 0
prefs["profile.password_manager_enabled"] = False
+ prefs["profile.password_manager_leak_detection"] = False
prefs["profile.default_content_setting_values.notifications"] = 2
prefs["profile.default_content_settings.popups"] = 0
prefs["profile.managed_default_content_settings.popups"] = 0
@@ -3271,6 +3461,7 @@ def get_local_driver(
"default_content_settings.popups": 0,
"managed_default_content_settings.popups": 0,
"profile.password_manager_enabled": False,
+ "profile.password_manager_leak_detection": False,
"profile.default_content_setting_values.notifications": 2,
"profile.default_content_settings.popups": 0,
"profile.managed_default_content_settings.popups": 0,
@@ -4244,13 +4435,17 @@ def get_local_driver(
with uc_lock: # Avoid multithreaded issues
if make_uc_driver_from_chromedriver:
if os.path.exists(LOCAL_CHROMEDRIVER):
- shutil.copyfile(
- LOCAL_CHROMEDRIVER, LOCAL_UC_DRIVER
- )
+ with suppress(Exception):
+ make_driver_executable_if_not(
+ LOCAL_CHROMEDRIVER
+ )
+ shutil.copy2(LOCAL_CHROMEDRIVER, LOCAL_UC_DRIVER)
elif os.path.exists(path_chromedriver):
- shutil.copyfile(
- path_chromedriver, LOCAL_UC_DRIVER
- )
+ with suppress(Exception):
+ make_driver_executable_if_not(
+ path_chromedriver
+ )
+ shutil.copy2(path_chromedriver, LOCAL_UC_DRIVER)
try:
make_driver_executable_if_not(LOCAL_UC_DRIVER)
except Exception as e:
@@ -4670,6 +4865,9 @@ def get_local_driver(
options=chrome_options,
)
driver.default_get = driver.get # Save copy of original
+ driver.cdp = None # Set a placeholder
+ driver._is_using_cdp = False
+ driver._is_connected = True
if uc_activated:
driver.get = lambda url: uc_special_open_if_cf(
driver,
@@ -4697,6 +4895,16 @@ def get_local_driver(
driver.uc_click = lambda *args, **kwargs: uc_click(
driver, *args, **kwargs
)
+ driver.uc_activate_cdp_mode = (
+ lambda *args, **kwargs: uc_activate_cdp_mode(
+ driver, *args, **kwargs
+ )
+ )
+ driver.uc_open_with_cdp_mode = (
+ lambda *args, **kwargs: uc_open_with_cdp_mode(
+ driver, *args, **kwargs
+ )
+ )
driver.uc_gui_press_key = (
lambda *args, **kwargs: uc_gui_press_key(
driver, *args, **kwargs
diff --git a/seleniumbase/core/sb_cdp.py b/seleniumbase/core/sb_cdp.py
new file mode 100644
index 00000000000..44aa07237d6
--- /dev/null
+++ b/seleniumbase/core/sb_cdp.py
@@ -0,0 +1,898 @@
+"""Add CDP methods to extend the driver"""
+import math
+import os
+import re
+import time
+from contextlib import suppress
+from seleniumbase import config as sb_config
+from seleniumbase.config import settings
+from seleniumbase.fixtures import constants
+from seleniumbase.fixtures import js_utils
+from seleniumbase.fixtures import page_utils
+from seleniumbase.fixtures import shared_utils
+
+
+class CDPMethods():
+ def __init__(self, loop, page, driver):
+ self.loop = loop
+ self.page = page
+ self.driver = driver
+
+ def __slow_mode_pause_if_set(self):
+ if hasattr(sb_config, "slow_mode") and sb_config.slow_mode:
+ time.sleep(0.16)
+
+ def __add_light_pause(self):
+ time.sleep(0.007)
+
+ def __convert_to_css_if_xpath(self, selector):
+ if page_utils.is_xpath_selector(selector):
+ with suppress(Exception):
+ css = js_utils.convert_to_css_selector(selector, "xpath")
+ if css:
+ return css
+ return selector
+
+ def __add_sync_methods(self, element):
+ if not element:
+ return element
+ element.clear_input = lambda: self.__clear_input(element)
+ element.click = lambda: self.__click(element)
+ element.flash = lambda: self.__flash(element)
+ element.focus = lambda: self.__focus(element)
+ element.highlight_overlay = lambda: self.__highlight_overlay(element)
+ element.mouse_click = lambda: self.__mouse_click(element)
+ element.mouse_drag = (
+ lambda destination: self.__mouse_drag(element, destination)
+ )
+ element.mouse_move = lambda: self.__mouse_move(element)
+ element.query_selector = (
+ lambda selector: self.__query_selector(element, selector)
+ )
+ element.querySelector = element.query_selector
+ element.query_selector_all = (
+ lambda selector: self.__query_selector_all(element, selector)
+ )
+ element.querySelectorAll = element.query_selector_all
+ element.remove_from_dom = lambda: self.__remove_from_dom(element)
+ element.save_screenshot = (
+ lambda *args, **kwargs: self.__save_screenshot(
+ element, *args, **kwargs)
+ )
+ element.save_to_dom = lambda: self.__save_to_dom(element)
+ element.scroll_into_view = lambda: self.__scroll_into_view(element)
+ element.select_option = lambda: self.__select_option(element)
+ element.send_file = (
+ lambda *file_paths: self.__send_file(element, *file_paths)
+ )
+ element.send_keys = lambda text: self.__send_keys(element, text)
+ element.set_text = lambda value: self.__set_text(element, value)
+ element.set_value = lambda value: self.__set_value(element, value)
+ element.type = lambda text: self.__type(element, text)
+ element.get_position = lambda: self.__get_position(element)
+ element.get_html = lambda: self.__get_html(element)
+ element.get_js_attributes = lambda: self.__get_js_attributes(element)
+ return element
+
+ def get(self, url):
+ url = shared_utils.fix_url_as_needed(url)
+ self.page = self.loop.run_until_complete(self.driver.cdp_base.get(url))
+ url_protocol = url.split(":")[0]
+ safe_url = True
+ if url_protocol not in ["about", "data", "chrome"]:
+ safe_url = False
+ if not safe_url:
+ time.sleep(constants.UC.CDP_MODE_OPEN_WAIT)
+
+ def reload(self, ignore_cache=True, script_to_evaluate_on_load=None):
+ self.loop.run_until_complete(
+ self.page.reload(
+ ignore_cache=ignore_cache,
+ script_to_evaluate_on_load=script_to_evaluate_on_load,
+ )
+ )
+
+ def refresh(self, *args, **kwargs):
+ self.reload(*args, **kwargs)
+
+ def get_event_loop(self):
+ return self.loop
+
+ def add_handler(self, event, handler):
+ self.page.add_handler(event, handler)
+
+ def find_element(
+ self, selector, best_match=False, timeout=settings.SMALL_TIMEOUT
+ ):
+ """Similar to select(), but also finds elements by text content.
+ When using text-based searches, if best_match=False, then will
+ find the first element with the text. If best_match=True, then
+ if multiple elements have that text, then will use the element
+ with the closest text-length to the text being searched for."""
+ self.__add_light_pause()
+ selector = self.__convert_to_css_if_xpath(selector)
+ if (":contains(" in selector):
+ tag_name = selector.split(":contains(")[0].split(" ")[-1]
+ text = selector.split(":contains(")[1].split(")")[0][1:-1]
+ with suppress(Exception):
+ self.loop.run_until_complete(
+ self.page.select(tag_name, timeout=3)
+ )
+ self.loop.run_until_complete(self.page.find(text, timeout=3))
+ element = self.find_elements_by_text(text, tag_name=tag_name)[0]
+ return self.__add_sync_methods(element)
+ failure = False
+ try:
+ element = self.loop.run_until_complete(
+ self.page.find(
+ selector, best_match=best_match, timeout=timeout
+ )
+ )
+ except Exception:
+ failure = True
+ plural = "s"
+ if timeout == 1:
+ plural = ""
+ message = "\n Element {%s} was not found after %s second%s!" % (
+ selector,
+ timeout,
+ plural,
+ )
+ if failure:
+ raise Exception(message)
+ element = self.__add_sync_methods(element)
+ self.__slow_mode_pause_if_set()
+ return element
+
+ def find_all(self, selector, timeout=settings.SMALL_TIMEOUT):
+ self.__add_light_pause()
+ selector = self.__convert_to_css_if_xpath(selector)
+ elements = self.loop.run_until_complete(
+ self.page.find_all(selector, timeout=timeout)
+ )
+ updated_elements = []
+ for element in elements:
+ element = self.__add_sync_methods(element)
+ updated_elements.append(element)
+ self.__slow_mode_pause_if_set()
+ return updated_elements
+
+ def find_elements_by_text(self, text, tag_name=None):
+ """Returns a list of elements by matching text.
+ Optionally, provide a tag_name to narrow down the search
+ to only elements with the given tag. (Eg: a, div, script, span)"""
+ self.__add_light_pause()
+ elements = self.loop.run_until_complete(
+ self.page.find_elements_by_text(text=text)
+ )
+ updated_elements = []
+ for element in elements:
+ if not tag_name or tag_name.lower() == element.tag_name.lower():
+ element = self.__add_sync_methods(element)
+ updated_elements.append(element)
+ self.__slow_mode_pause_if_set()
+ return updated_elements
+
+ def select(self, selector, timeout=settings.SMALL_TIMEOUT):
+ """Similar to find_element(), but without text-based search."""
+ self.__add_light_pause()
+ selector = self.__convert_to_css_if_xpath(selector)
+ if (":contains(" in selector):
+ tag_name = selector.split(":contains(")[0].split(" ")[-1]
+ text = selector.split(":contains(")[1].split(")")[0][1:-1]
+ with suppress(Exception):
+ self.loop.run_until_complete(
+ self.page.select(tag_name, timeout=5)
+ )
+ self.loop.run_until_complete(self.page.find(text, timeout=5))
+ element = self.find_elements_by_text(text, tag_name=tag_name)[0]
+ return self.__add_sync_methods(element)
+ failure = False
+ try:
+ element = self.loop.run_until_complete(
+ self.page.select(selector, timeout=timeout)
+ )
+ except Exception:
+ failure = True
+ plural = "s"
+ if timeout == 1:
+ plural = ""
+ message = "\n Element {%s} was not found after %s second%s!" % (
+ selector,
+ timeout,
+ plural,
+ )
+ if failure:
+ raise Exception(message)
+ element = self.__add_sync_methods(element)
+ self.__slow_mode_pause_if_set()
+ return element
+
+ def select_all(self, selector, timeout=settings.SMALL_TIMEOUT):
+ self.__add_light_pause()
+ selector = self.__convert_to_css_if_xpath(selector)
+ elements = self.loop.run_until_complete(
+ self.page.select_all(selector, timeout=timeout)
+ )
+ updated_elements = []
+ for element in elements:
+ element = self.__add_sync_methods(element)
+ updated_elements.append(element)
+ self.__slow_mode_pause_if_set()
+ return updated_elements
+
+ def click_link(self, link_text):
+ self.find_elements_by_text(link_text, "a")[0].click()
+
+ def __clear_input(self, element):
+ return (
+ self.loop.run_until_complete(element.clear_input_async())
+ )
+
+ def __click(self, element):
+ return (
+ self.loop.run_until_complete(element.click_async())
+ )
+
+ def __flash(self, element):
+ return (
+ self.loop.run_until_complete(element.flash_async())
+ )
+
+ def __focus(self, element):
+ return (
+ self.loop.run_until_complete(element.focus_async())
+ )
+
+ def __highlight_overlay(self, element):
+ return (
+ self.loop.run_until_complete(element.highlight_overlay_async())
+ )
+
+ def __mouse_click(self, element):
+ return (
+ self.loop.run_until_complete(element.mouse_click_async())
+ )
+
+ def __mouse_drag(self, element, destination):
+ return (
+ self.loop.run_until_complete(element.mouse_drag_async(destination))
+ )
+
+ def __mouse_move(self, element):
+ return (
+ self.loop.run_until_complete(element.mouse_move_async())
+ )
+
+ def __query_selector(self, element, selector):
+ selector = self.__convert_to_css_if_xpath(selector)
+ element = self.loop.run_until_complete(
+ element.query_selector_async(selector)
+ )
+ element = self.__add_sync_methods(element)
+ return element
+
+ def __query_selector_all(self, element, selector):
+ selector = self.__convert_to_css_if_xpath(selector)
+ elements = self.loop.run_until_complete(
+ element.query_selector_all_async(selector)
+ )
+ updated_elements = []
+ for element in elements:
+ element = self.__add_sync_methods(element)
+ updated_elements.append(element)
+ self.__slow_mode_pause_if_set()
+ return updated_elements
+
+ def __remove_from_dom(self, element):
+ return (
+ self.loop.run_until_complete(element.remove_from_dom_async())
+ )
+
+ def __save_screenshot(self, element, *args, **kwargs):
+ return (
+ self.loop.run_until_complete(
+ element.save_screenshot_async(*args, **kwargs)
+ )
+ )
+
+ def __save_to_dom(self, element):
+ return (
+ self.loop.run_until_complete(element.save_to_dom_async())
+ )
+
+ def __scroll_into_view(self, element):
+ return (
+ self.loop.run_until_complete(element.scroll_into_view_async())
+ )
+
+ def __select_option(self, element):
+ return (
+ self.loop.run_until_complete(element.select_option_async())
+ )
+
+ def __send_file(self, element, *file_paths):
+ return (
+ self.loop.run_until_complete(element.send_file_async(*file_paths))
+ )
+
+ def __send_keys(self, element, text):
+ return (
+ self.loop.run_until_complete(element.send_keys_async(text))
+ )
+
+ def __set_text(self, element, value):
+ return (
+ self.loop.run_until_complete(element.set_text_async(value))
+ )
+
+ def __set_value(self, element, value):
+ return (
+ self.loop.run_until_complete(element.set_value_async(value))
+ )
+
+ def __type(self, element, text):
+ with suppress(Exception):
+ element.clear_input()
+ element.send_keys(text)
+
+ def __get_position(self, element):
+ return (
+ self.loop.run_until_complete(element.get_position_async())
+ )
+
+ def __get_html(self, element):
+ return (
+ self.loop.run_until_complete(element.get_html_async())
+ )
+
+ def __get_js_attributes(self, element):
+ return (
+ self.loop.run_until_complete(element.get_js_attributes_async())
+ )
+
+ def tile_windows(self, windows=None, max_columns=0):
+ """Tile windows and return the grid of tiled windows."""
+ return self.loop.run_until_complete(
+ self.driver.cdp_base.tile_windows(windows, max_columns)
+ )
+
+ def get_all_cookies(self, *args, **kwargs):
+ return self.loop.run_until_complete(
+ self.driver.cdp_base.cookies.get_all(*args, **kwargs)
+ )
+
+ def set_all_cookies(self, *args, **kwargs):
+ return self.loop.run_until_complete(
+ self.driver.cdp_base.cookies.set_all(*args, **kwargs)
+ )
+
+ def save_cookies(self, *args, **kwargs):
+ return self.loop.run_until_complete(
+ self.driver.cdp_base.cookies.save(*args, **kwargs)
+ )
+
+ def load_cookies(self, *args, **kwargs):
+ return self.loop.run_until_complete(
+ self.driver.cdp_base.cookies.load(*args, **kwargs)
+ )
+
+ def clear_cookies(self, *args, **kwargs):
+ return self.loop.run_until_complete(
+ self.driver.cdp_base.cookies.clear(*args, **kwargs)
+ )
+
+ def sleep(self, seconds):
+ time.sleep(seconds)
+
+ def bring_active_window_to_front(self):
+ self.loop.run_until_complete(self.page.bring_to_front())
+
+ def get_active_element(self):
+ return self.loop.run_until_complete(
+ self.page.js_dumps("document.activeElement")
+ )
+
+ def get_active_element_css(self):
+ from seleniumbase.js_code import active_css_js
+
+ js_code = active_css_js.get_active_element_css
+ js_code = js_code.replace("return getBestSelector", "getBestSelector")
+ return self.loop.run_until_complete(
+ self.page.evaluate(js_code)
+ )
+
+ def click(self, selector, timeout=settings.SMALL_TIMEOUT):
+ self.__slow_mode_pause_if_set()
+ element = self.find_element(selector, timeout=timeout)
+ self.__add_light_pause()
+ element.click()
+ self.__slow_mode_pause_if_set()
+
+ def click_active_element(self):
+ self.loop.run_until_complete(
+ self.page.evaluate("document.activeElement.click()")
+ )
+ self.__slow_mode_pause_if_set()
+
+ def click_if_visible(self, selector):
+ if self.is_element_visible(selector):
+ self.find_element(selector).click()
+ self.__slow_mode_pause_if_set()
+
+ def mouse_click(self, selector, timeout=settings.SMALL_TIMEOUT):
+ """(Attempt simulating a mouse click)"""
+ self.__slow_mode_pause_if_set()
+ element = self.find_element(selector, timeout=timeout)
+ self.__add_light_pause()
+ element.mouse_click()
+ self.__slow_mode_pause_if_set()
+
+ def nested_click(self, parent_selector, selector):
+ """
+ Find parent element and click on child element inside it.
+ (This can be used to click on elements inside an iframe.)
+ """
+ element = self.find_element(parent_selector)
+ element.query_selector(selector).mouse_click()
+ self.__slow_mode_pause_if_set()
+
+ def get_nested_element(self, parent_selector, selector):
+ """(Can be used to find an element inside an iframe)"""
+ element = self.find_element(parent_selector)
+ return element.query_selector(selector)
+
+ def flash(self, selector):
+ """Paint a quickly-vanishing dot over an element."""
+ self.find_element(selector).flash()
+
+ def focus(self, selector):
+ self.find_element(selector).focus()
+
+ def highlight_overlay(self, selector):
+ self.find_element(selector).highlight_overlay()
+
+ def remove_element(self, selector):
+ self.select(selector).remove_from_dom()
+
+ def remove_from_dom(self, selector):
+ self.select(selector).remove_from_dom()
+
+ def remove_elements(self, selector):
+ """Remove all elements on the page that match the selector."""
+ css_selector = self.__convert_to_css_if_xpath(selector)
+ css_selector = re.escape(css_selector) # Add "\\" to special chars
+ css_selector = js_utils.escape_quotes_if_needed(css_selector)
+ js_code = (
+ """var $elements = document.querySelectorAll('%s');
+ var index = 0, length = $elements.length;
+ for(; index < length; index++){
+ $elements[index].remove();}"""
+ % css_selector
+ )
+ with suppress(Exception):
+ self.loop.run_until_complete(self.page.evaluate(js_code))
+
+ def scroll_into_view(self, selector):
+ self.find_element(selector).scroll_into_view()
+
+ def send_keys(self, selector, text, timeout=settings.SMALL_TIMEOUT):
+ element = self.select(selector)
+ self.__slow_mode_pause_if_set()
+ if text.endswith("\n") or text.endswith("\r"):
+ text = text[:-1] + "\r\n"
+ element.send_keys(text)
+ self.__slow_mode_pause_if_set()
+
+ def press_keys(self, selector, text, timeout=settings.SMALL_TIMEOUT):
+ """Similar to send_keys(), but presses keys at human speed."""
+ element = self.select(selector)
+ self.__slow_mode_pause_if_set()
+ submit = False
+ if text.endswith("\n") or text.endswith("\r"):
+ submit = True
+ text = text[:-1]
+ for key in text:
+ element.send_keys(key)
+ time.sleep(0.0375)
+ if submit:
+ element.send_keys("\r\n")
+ time.sleep(0.0375)
+ self.__slow_mode_pause_if_set()
+
+ def type(self, selector, text, timeout=settings.SMALL_TIMEOUT):
+ """Similar to send_keys(), but clears the text field first."""
+ element = self.select(selector)
+ self.__slow_mode_pause_if_set()
+ with suppress(Exception):
+ element.clear_input()
+ if text.endswith("\n") or text.endswith("\r"):
+ text = text[:-1] + "\r\n"
+ element.send_keys(text)
+ self.__slow_mode_pause_if_set()
+
+ def evaluate(self, expression):
+ """Run a JavaScript expression and return the result."""
+ return self.loop.run_until_complete(
+ self.page.evaluate(expression)
+ )
+
+ def js_dumps(self, obj_name):
+ """Similar to evaluate(), but for dictionary results."""
+ return self.loop.run_until_complete(
+ self.page.js_dumps(obj_name)
+ )
+
+ def maximize(self):
+ return self.loop.run_until_complete(
+ self.page.maximize()
+ )
+
+ def minimize(self):
+ return self.loop.run_until_complete(
+ self.page.minimize()
+ )
+
+ def medimize(self):
+ return self.loop.run_until_complete(
+ self.page.medimize()
+ )
+
+ def set_window_rect(self, x, y, width, height):
+ return self.loop.run_until_complete(
+ self.page.set_window_size(
+ left=x, top=y, width=width, height=height)
+ )
+
+ def reset_window_size(self):
+ x = settings.WINDOW_START_X
+ y = settings.WINDOW_START_Y
+ width = settings.CHROME_START_WIDTH
+ height = settings.CHROME_START_HEIGHT
+ self.set_window_rect(x, y, width, height)
+
+ def get_window(self):
+ return self.loop.run_until_complete(
+ self.page.get_window()
+ )
+
+ def get_text(self, selector):
+ return self.find_element(selector).text_all
+
+ def get_title(self):
+ return self.loop.run_until_complete(
+ self.page.evaluate("document.title")
+ )
+
+ def get_current_url(self):
+ return self.loop.run_until_complete(
+ self.page.evaluate("window.location.href")
+ )
+
+ def get_origin(self):
+ return self.loop.run_until_complete(
+ self.page.evaluate("window.location.origin")
+ )
+
+ def get_page_source(self):
+ try:
+ source = self.loop.run_until_complete(
+ self.page.evaluate("document.documentElement.outerHTML")
+ )
+ except Exception:
+ time.sleep(constants.UC.CDP_MODE_OPEN_WAIT)
+ source = self.loop.run_until_complete(
+ self.page.evaluate("document.documentElement.outerHTML")
+ )
+ return source
+
+ def get_user_agent(self):
+ return self.loop.run_until_complete(
+ self.page.evaluate("navigator.userAgent")
+ )
+
+ def get_cookie_string(self):
+ return self.loop.run_until_complete(
+ self.page.evaluate("document.cookie")
+ )
+
+ def get_locale_code(self):
+ return self.loop.run_until_complete(
+ self.page.evaluate("navigator.language || navigator.languages[0]")
+ )
+
+ def get_screen_rect(self):
+ coordinates = self.loop.run_until_complete(
+ self.page.js_dumps("window.screen")
+ )
+ return coordinates
+
+ def get_window_rect(self):
+ coordinates = {}
+ innerWidth = self.loop.run_until_complete(
+ self.page.evaluate("window.innerWidth")
+ )
+ innerHeight = self.loop.run_until_complete(
+ self.page.evaluate("window.innerHeight")
+ )
+ outerWidth = self.loop.run_until_complete(
+ self.page.evaluate("window.outerWidth")
+ )
+ outerHeight = self.loop.run_until_complete(
+ self.page.evaluate("window.outerHeight")
+ )
+ pageXOffset = self.loop.run_until_complete(
+ self.page.evaluate("window.pageXOffset")
+ )
+ pageYOffset = self.loop.run_until_complete(
+ self.page.evaluate("window.pageYOffset")
+ )
+ scrollX = self.loop.run_until_complete(
+ self.page.evaluate("window.scrollX")
+ )
+ scrollY = self.loop.run_until_complete(
+ self.page.evaluate("window.scrollY")
+ )
+ screenLeft = self.loop.run_until_complete(
+ self.page.evaluate("window.screenLeft")
+ )
+ screenTop = self.loop.run_until_complete(
+ self.page.evaluate("window.screenTop")
+ )
+ x = self.loop.run_until_complete(
+ self.page.evaluate("window.screenX")
+ )
+ y = self.loop.run_until_complete(
+ self.page.evaluate("window.screenY")
+ )
+ coordinates["innerWidth"] = innerWidth
+ coordinates["innerHeight"] = innerHeight
+ coordinates["outerWidth"] = outerWidth
+ coordinates["outerHeight"] = outerHeight
+ coordinates["width"] = outerWidth
+ coordinates["height"] = outerHeight
+ coordinates["pageXOffset"] = pageXOffset if pageXOffset else 0
+ coordinates["pageYOffset"] = pageYOffset if pageYOffset else 0
+ coordinates["scrollX"] = scrollX if scrollX else 0
+ coordinates["scrollY"] = scrollY if scrollY else 0
+ coordinates["screenLeft"] = screenLeft if screenLeft else 0
+ coordinates["screenTop"] = screenTop if screenTop else 0
+ coordinates["x"] = x if x else 0
+ coordinates["y"] = y if y else 0
+ return coordinates
+
+ def get_window_size(self):
+ coordinates = {}
+ outerWidth = self.loop.run_until_complete(
+ self.page.evaluate("window.outerWidth")
+ )
+ outerHeight = self.loop.run_until_complete(
+ self.page.evaluate("window.outerHeight")
+ )
+ coordinates["width"] = outerWidth
+ coordinates["height"] = outerHeight
+ return coordinates
+
+ def get_window_position(self):
+ coordinates = {}
+ x = self.loop.run_until_complete(
+ self.page.evaluate("window.screenX")
+ )
+ y = self.loop.run_until_complete(
+ self.page.evaluate("window.screenY")
+ )
+ coordinates["x"] = x if x else 0
+ coordinates["y"] = y if y else 0
+ return coordinates
+
+ def get_element_rect(self, selector):
+ selector = self.__convert_to_css_if_xpath(selector)
+ coordinates = self.loop.run_until_complete(
+ self.page.js_dumps(
+ """document.querySelector"""
+ """('%s').getBoundingClientRect()"""
+ % js_utils.escape_quotes_if_needed(re.escape(selector))
+ )
+ )
+ return coordinates
+
+ def get_element_size(self, selector):
+ element_rect = self.get_element_rect(selector)
+ coordinates = {}
+ coordinates["width"] = element_rect["width"]
+ coordinates["height"] = element_rect["height"]
+ return coordinates
+
+ def get_element_position(self, selector):
+ element_rect = self.get_element_rect(selector)
+ coordinates = {}
+ coordinates["x"] = element_rect["x"]
+ coordinates["y"] = element_rect["y"]
+ return coordinates
+
+ def get_gui_element_rect(self, selector):
+ """(Coordinates are relative to the screen. Not the window.)"""
+ element_rect = self.get_element_rect(selector)
+ e_width = element_rect["width"]
+ e_height = element_rect["height"]
+ window_rect = self.get_window_rect()
+ w_bottom_y = window_rect["y"] + window_rect["height"]
+ viewport_height = window_rect["innerHeight"]
+ x = math.ceil(window_rect["x"] + element_rect["x"])
+ y = math.ceil(w_bottom_y - viewport_height + element_rect["y"])
+ y_scroll_offset = window_rect["pageYOffset"]
+ y = int(y - y_scroll_offset)
+ return ({"height": e_height, "width": e_width, "x": x, "y": y})
+
+ def get_gui_element_center(self, selector):
+ """(Coordinates are relative to the screen. Not the window.)"""
+ element_rect = self.get_gui_element_rect(selector)
+ e_width = element_rect["width"]
+ e_height = element_rect["height"]
+ e_x = element_rect["x"]
+ e_y = element_rect["y"]
+ return ((e_x + e_width / 2), (e_y + e_height / 2))
+
+ def get_document(self):
+ return self.loop.run_until_complete(
+ self.page.get_document()
+ )
+
+ def get_flattened_document(self):
+ return self.loop.run_until_complete(
+ self.page.get_flattened_document()
+ )
+
+ def get_element_attributes(self, selector):
+ return self.loop.run_until_complete(
+ self.page.js_dumps(
+ """document.querySelector('%s')"""
+ % js_utils.escape_quotes_if_needed(re.escape(selector))
+ )
+ )
+
+ def get_element_html(self, selector):
+ selector = self.__convert_to_css_if_xpath(selector)
+ return self.loop.run_until_complete(
+ self.page.evaluate(
+ """document.querySelector('%s').outerHTML"""
+ % js_utils.escape_quotes_if_needed(re.escape(selector))
+ )
+ )
+
+ def set_attributes(self, selector, attribute, value):
+ """This method uses JavaScript to set/update a common attribute.
+ All matching selectors from querySelectorAll() are used.
+ Example => (Make all links on a website redirect to Google):
+ self.set_attributes("a", "href", "https://google.com")"""
+ attribute = re.escape(attribute)
+ attribute = js_utils.escape_quotes_if_needed(attribute)
+ value = re.escape(value)
+ value = js_utils.escape_quotes_if_needed(value)
+ css_selector = self.__convert_to_css_if_xpath(selector)
+ css_selector = re.escape(css_selector) # Add "\\" to special chars
+ css_selector = js_utils.escape_quotes_if_needed(css_selector)
+ js_code = """var $elements = document.querySelectorAll('%s');
+ var index = 0, length = $elements.length;
+ for(; index < length; index++){
+ $elements[index].setAttribute('%s','%s');}""" % (
+ css_selector,
+ attribute,
+ value,
+ )
+ with suppress(Exception):
+ self.loop.run_until_complete(self.page.evaluate(js_code))
+
+ def internalize_links(self):
+ """All `target="_blank"` links become `target="_self"`.
+ This prevents those links from opening in a new tab."""
+ self.set_attributes('[target="_blank"]', "target", "_self")
+
+ def is_element_present(self, selector):
+ try:
+ self.select(selector, timeout=0.01)
+ return True
+ except Exception:
+ return False
+ selector = self.__convert_to_css_if_xpath(selector)
+ element = self.loop.run_until_complete(
+ self.page.js_dumps(
+ """document.querySelector('%s')"""
+ % js_utils.escape_quotes_if_needed(re.escape(selector))
+ )
+ )
+ return element is not None
+
+ def is_element_visible(self, selector):
+ selector = self.__convert_to_css_if_xpath(selector)
+ element = None
+ if ":contains(" not in selector:
+ try:
+ element = self.loop.run_until_complete(
+ self.page.js_dumps(
+ """window.getComputedStyle(document.querySelector"""
+ """('%s'))"""
+ % js_utils.escape_quotes_if_needed(re.escape(selector))
+ )
+ )
+ except Exception:
+ return False
+ if not element:
+ return False
+ return element.get("display") != "none"
+ else:
+ with suppress(Exception):
+ tag_name = selector.split(":contains(")[0].split(" ")[-1]
+ text = selector.split(":contains(")[1].split(")")[0][1:-1]
+ self.loop.run_until_complete(
+ self.page.select(tag_name, timeout=0.1)
+ )
+ self.loop.run_until_complete(self.page.find(text, timeout=0.1))
+ return True
+ return False
+
+ def assert_element(self, selector, timeout=settings.SMALL_TIMEOUT):
+ try:
+ self.select(selector, timeout=timeout)
+ except Exception:
+ raise Exception("Element {%s} not found!" % selector)
+ for i in range(30):
+ if self.is_element_visible(selector):
+ return True
+ time.sleep(0.1)
+ raise Exception("Element {%s} not visible!" % selector)
+
+ def assert_element_present(self, selector, timeout=settings.SMALL_TIMEOUT):
+ try:
+ self.select(selector, timeout=timeout)
+ except Exception:
+ raise Exception("Element {%s} not found!" % selector)
+ return True
+
+ def assert_text(
+ self, text, selector="html", timeout=settings.SMALL_TIMEOUT
+ ):
+ element = None
+ try:
+ element = self.select(selector, timeout=timeout)
+ except Exception:
+ raise Exception("Element {%s} not found!" % selector)
+ for i in range(30):
+ if self.is_element_visible(selector) and text in element.text_all:
+ return True
+ time.sleep(0.1)
+ raise Exception(
+ "Text {%s} not found in {%s}! Actual text: {%s}"
+ % (text, selector, element.text_all)
+ )
+
+ def assert_exact_text(
+ self, text, selector="html", timeout=settings.SMALL_TIMEOUT
+ ):
+ element = None
+ try:
+ element = self.select(selector, timeout=timeout)
+ except Exception:
+ raise Exception("Element {%s} not found!" % selector)
+ for i in range(30):
+ if (
+ self.is_element_visible(selector)
+ and text.strip() == element.text_all.strip()
+ ):
+ return True
+ time.sleep(0.1)
+ raise Exception(
+ "Expected Text {%s}, is not equal to {%s} in {%s}!"
+ % (text, element.text_all, selector)
+ )
+
+ def save_screenshot(self, name, folder=None, selector=None):
+ filename = name
+ if folder:
+ filename = os.path.join(folder, name)
+ if not selector:
+ self.loop.run_until_complete(
+ self.page.save_screenshot(filename)
+ )
+ else:
+ self.select(selector).save_screenshot(filename)
diff --git a/seleniumbase/core/sb_driver.py b/seleniumbase/core/sb_driver.py
index 6fc9dea1730..d4baab0b4be 100644
--- a/seleniumbase/core/sb_driver.py
+++ b/seleniumbase/core/sb_driver.py
@@ -4,12 +4,17 @@
from seleniumbase.fixtures import js_utils
from seleniumbase.fixtures import page_actions
from seleniumbase.fixtures import page_utils
+from seleniumbase.fixtures import shared_utils
class DriverMethods():
def __init__(self, driver):
self.driver = driver
+ def __is_cdp_swap_needed(self):
+ """If the driver is disconnected, use a CDP method when available."""
+ return shared_utils.is_cdp_swap_needed(self.driver)
+
def find_element(self, by=None, value=None):
if not value:
value = by
@@ -45,10 +50,21 @@ def get_attribute(self, selector, attribute, by="css selector"):
element = self.locator(selector, by=by)
return element.get_attribute(attribute)
+ def get_current_url(self):
+ if self.__is_cdp_swap_needed():
+ current_url = self.driver.cdp.get_current_url()
+ else:
+ current_url = self.driver.current_url
+ return current_url
+
def get_page_source(self):
+ if self.__is_cdp_swap_needed():
+ return self.driver.cdp.get_page_source()
return self.driver.page_source
def get_title(self):
+ if self.__is_cdp_swap_needed():
+ return self.driver.cdp.get_title()
return self.driver.title
def open_url(self, *args, **kwargs):
@@ -175,6 +191,34 @@ def is_alert_present(self):
def is_online(self):
return self.driver.execute_script("return navigator.onLine;")
+ def is_connected(self):
+ """
+ Return True if WebDriver is connected to the browser.
+ Note that the stealthy CDP-Driver isn't a WebDriver.
+ In CDP Mode, the CDP-Driver controls the web browser.
+ The CDP-Driver can be connected while WebDriver isn't.
+ """
+ try:
+ self.driver.window_handles
+ return True
+ except Exception:
+ return False
+
+ def is_uc_mode_active(self):
+ """Return True if the driver is using UC Mode. False otherwise."""
+ return (
+ hasattr(self.driver, "_is_using_uc")
+ and self.driver._is_using_uc
+ )
+
+ def is_cdp_mode_active(self):
+ """CDP Mode is a special mode within UC Mode. Activated separately.
+ Return True if CDP Mode is loaded in the driver. False otherwise."""
+ return (
+ hasattr(self.driver, "_is_using_cdp")
+ and self.driver._is_using_cdp
+ )
+
def js_click(self, *args, **kwargs):
return page_actions.js_click(self.driver, *args, **kwargs)
diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py
index f97a1cc5aff..4007c3fe606 100644
--- a/seleniumbase/fixtures/base_case.py
+++ b/seleniumbase/fixtures/base_case.py
@@ -223,6 +223,9 @@ def test_example(self):
def open(self, url):
"""Navigates the current browser window to the specified page."""
self.__check_scope()
+ if self.__is_cdp_swap_needed():
+ self.cdp.open(url)
+ return
self._check_browser()
if self.__needs_minimum_wait():
time.sleep(0.04)
@@ -388,6 +391,9 @@ def click(
original_selector = selector
original_by = by
selector, by = self.__recalculate_selector(selector, by)
+ if self.__is_cdp_swap_needed():
+ self.cdp.click(selector)
+ return
if delay and (type(delay) in [int, float]) and delay > 0:
time.sleep(delay)
if page_utils.is_link_text_selector(selector) or by == By.LINK_TEXT:
@@ -878,6 +884,9 @@ def update_text(
if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:
timeout = self.__get_new_timeout(timeout)
selector, by = self.__recalculate_selector(selector, by)
+ if self.__is_cdp_swap_needed():
+ self.cdp.type(selector, text)
+ return
if self.__is_shadow_selector(selector):
self.__shadow_type(selector, text, timeout)
return
@@ -991,6 +1000,9 @@ def add_text(self, selector, text, by="css selector", timeout=None):
if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:
timeout = self.__get_new_timeout(timeout)
selector, by = self.__recalculate_selector(selector, by)
+ if self.__is_cdp_swap_needed():
+ self.cdp.send_keys(selector, text)
+ return
if self.__is_shadow_selector(selector):
self.__shadow_type(selector, text, timeout, clear_first=False)
return
@@ -1099,6 +1111,9 @@ def send_keys(self, selector, text, by="css selector", timeout=None):
def press_keys(self, selector, text, by="css selector", timeout=None):
"""Use send_keys() to press one key at a time."""
+ if self.__is_cdp_swap_needed():
+ self.cdp.press_keys(selector, text)
+ return
self.wait_for_ready_state_complete()
element = self.wait_for_element_present(
selector, by=by, timeout=timeout
@@ -1207,6 +1222,9 @@ def focus(self, selector, by="css selector", timeout=None):
if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:
timeout = self.__get_new_timeout(timeout)
selector, by = self.__recalculate_selector(selector, by)
+ if self.__is_cdp_swap_needed():
+ self.cdp.focus(selector)
+ return
element = self.wait_for_element_present(
selector, by=by, timeout=timeout
)
@@ -1237,6 +1255,9 @@ def focus(self, selector, by="css selector", timeout=None):
def refresh_page(self):
self.__check_scope()
self.__last_page_load_url = None
+ if self.__is_cdp_swap_needed():
+ self.cdp.reload()
+ return
js_utils.clear_out_console_logs(self.driver)
self.driver.refresh()
self.wait_for_ready_state_complete()
@@ -1247,7 +1268,11 @@ def refresh(self):
def get_current_url(self):
self.__check_scope()
- current_url = self.driver.current_url
+ current_url = None
+ if self.__is_cdp_swap_needed():
+ current_url = self.cdp.get_current_url()
+ else:
+ current_url = self.driver.current_url
if "%" in current_url:
try:
from urllib.parse import unquote
@@ -1262,15 +1287,22 @@ def get_origin(self):
return self.execute_script("return window.location.origin;")
def get_page_source(self):
+ if self.__is_cdp_swap_needed():
+ return self.cdp.get_page_source()
self.wait_for_ready_state_complete()
if self.__needs_minimum_wait:
- time.sleep(0.02)
+ time.sleep(0.025)
return self.driver.page_source
def get_page_title(self):
+ if self.__is_cdp_swap_needed():
+ return self.cdp.get_title()
self.wait_for_ready_state_complete()
- self.wait_for_element_present("title", timeout=settings.SMALL_TIMEOUT)
- time.sleep(0.03)
+ with suppress(Exception):
+ self.wait_for_element_present(
+ "title", by="css selector", timeout=settings.MINI_TIMEOUT
+ )
+ time.sleep(0.025)
return self.driver.title
def get_title(self):
@@ -1365,7 +1397,7 @@ def open_if_not_url(self, url):
to convert the open() action into open_if_not_url() so that the
same page isn't opened again if the user is already on the page."""
self.__check_scope()
- current_url = self.driver.current_url
+ current_url = self.get_current_url()
if current_url != url:
if (
"?q=" not in current_url
@@ -1377,6 +1409,8 @@ def open_if_not_url(self, url):
def is_element_present(self, selector, by="css selector"):
"""Returns whether the element exists in the HTML."""
+ if self.__is_cdp_swap_needed():
+ return self.cdp.is_element_present(selector)
self.wait_for_ready_state_complete()
selector, by = self.__recalculate_selector(selector, by)
if self.__is_shadow_selector(selector):
@@ -1385,6 +1419,8 @@ def is_element_present(self, selector, by="css selector"):
def is_element_visible(self, selector, by="css selector"):
"""Returns whether the element is visible on the page."""
+ if self.__is_cdp_swap_needed():
+ return self.cdp.is_element_visible(selector)
self.wait_for_ready_state_complete()
selector, by = self.__recalculate_selector(selector, by)
if self.__is_shadow_selector(selector):
@@ -1562,6 +1598,9 @@ def click_link_text(self, link_text, timeout=None):
if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:
timeout = self.__get_new_timeout(timeout)
link_text = self.__get_type_checked_text(link_text)
+ if self.__is_cdp_swap_needed():
+ self.cdp.click_link(link_text)
+ return
if self.browser == "safari":
if self.demo_mode:
self.wait_for_link_text_present(link_text, timeout=timeout)
@@ -1794,6 +1833,8 @@ def get_text(self, selector="html", by="css selector", timeout=None):
if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:
timeout = self.__get_new_timeout(timeout)
selector, by = self.__recalculate_selector(selector, by)
+ if self.__is_cdp_swap_needed():
+ return self.cdp.get_text(selector)
if self.__is_shadow_selector(selector):
return self.__get_shadow_text(selector, timeout)
self.wait_for_ready_state_complete()
@@ -1921,6 +1962,9 @@ def set_attributes(self, selector, attribute, value, by="css selector"):
self.set_attributes("a", "href", "https://google.com")"""
self.__check_scope()
selector, by = self.__recalculate_selector(selector, by)
+ if self.__is_cdp_swap_needed():
+ self.cdp.set_attributes(selector, attribute, value)
+ return
original_attribute = attribute
original_value = value
attribute = re.escape(attribute)
@@ -2001,6 +2045,14 @@ def remove_attributes(self, selector, attribute, by="css selector"):
with suppress(Exception):
self.execute_script(script)
+ def internalize_links(self):
+ """All `target="_blank"` links become `target="_self"`.
+ This prevents those links from opening in a new tab."""
+ if self.__is_cdp_swap_needed():
+ self.cdp.internalize_links()
+ return
+ self.set_attributes('[target="_blank"]', "target", "_self")
+
def get_property(
self, selector, property, by="css selector", timeout=None
):
@@ -2104,6 +2156,12 @@ def find_elements(self, selector, by="css selector", limit=0):
Elements could be either hidden or visible on the page.
If "limit" is set and > 0, will only return that many elements."""
selector, by = self.__recalculate_selector(selector, by)
+ if self.__is_cdp_swap_needed():
+ elements = self.cdp.select_all(selector)
+ if limit and limit > 0 and len(elements) > limit:
+ elements = elements[:limit]
+ return elements
+
self.wait_for_ready_state_complete()
time.sleep(0.05)
elements = self.driver.find_elements(by=by, value=selector)
@@ -2277,6 +2335,9 @@ def click_if_visible(self, selector, by="css selector", timeout=0):
Use click_visible_elements() to click all matching elements.
If a "timeout" is provided, waits that long for the element
to appear before giving up and returning without a click()."""
+ if self.__is_cdp_swap_needed():
+ self.cdp.click_if_visible(selector)
+ return
self.wait_for_ready_state_complete()
if self.is_element_visible(selector, by=by):
self.click(selector, by=by)
@@ -2289,6 +2350,9 @@ def click_if_visible(self, selector, by="css selector", timeout=0):
self.click(selector, by=by)
def click_active_element(self):
+ if self.__is_cdp_swap_needed():
+ self.cdp.click_active_element()
+ return
self.wait_for_ready_state_complete()
pre_action_url = None
with suppress(Exception):
@@ -3311,6 +3375,8 @@ def get_gui_element_rect(self, selector, by="css selector"):
relative to the entire screen, rather than the browser window.
This is specifically for PyAutoGUI actions on the full screen.
(Note: There may be complications if iframes are involved.)"""
+ if self.__is_cdp_swap_needed():
+ return self.cdp.get_gui_element_rect(selector)
element = self.wait_for_element_present(selector, by=by, timeout=1)
element_rect = element.rect
e_width = element_rect["width"]
@@ -3344,6 +3410,8 @@ def get_gui_element_center(self, selector, by="css selector"):
on the entire GUI / screen, rather than on the browser window.
This is specifically for PyAutoGUI actions on the full screen.
(Note: There may be complications if iframes are involved.)"""
+ if self.__is_cdp_swap_needed():
+ return self.cdp.get_gui_element_center(selector)
element_rect = self.get_gui_element_rect(selector, by=by)
x = int(element_rect["x"]) + int(element_rect["width"] / 2) + 1
y = int(element_rect["y"]) + int(element_rect["height"] / 2) + 1
@@ -3351,16 +3419,22 @@ def get_gui_element_center(self, selector, by="css selector"):
def get_window_rect(self):
self.__check_scope()
+ if self.__is_cdp_swap_needed():
+ return self.cdp.get_window_rect()
self._check_browser()
return self.driver.get_window_rect()
def get_window_size(self):
self.__check_scope()
+ if self.__is_cdp_swap_needed():
+ return self.cdp.get_window_size()
self._check_browser()
return self.driver.get_window_size()
def get_window_position(self):
self.__check_scope()
+ if self.__is_cdp_swap_needed():
+ return self.cdp.get_window_position()
self._check_browser()
return self.driver.get_window_position()
@@ -4138,12 +4212,16 @@ def get_new_driver(
self.open(new_start_page)
self.__dont_record_open = False
if undetectable:
+ if hasattr(new_driver, "cdp"):
+ self.cdp = new_driver.cdp
if hasattr(new_driver, "uc_open"):
self.uc_open = new_driver.uc_open
if hasattr(new_driver, "uc_open_with_tab"):
self.uc_open_with_tab = new_driver.uc_open_with_tab
if hasattr(new_driver, "uc_open_with_reconnect"):
self.uc_open_with_reconnect = new_driver.uc_open_with_reconnect
+ if hasattr(new_driver, "uc_open_with_cdp_mode"):
+ self.uc_open_with_cdp_mode = new_driver.uc_open_with_cdp_mode
if hasattr(new_driver, "uc_open_with_disconnect"):
self.uc_open_with_disconnect = (
new_driver.uc_open_with_disconnect
@@ -4207,6 +4285,9 @@ def save_screenshot(
If a provided selector is not found, then takes a full-page screenshot.
If the folder provided doesn't exist, it will get created.
The screenshot will be in PNG format: (*.png)"""
+ if self.__is_cdp_swap_needed():
+ self.cdp.save_screenshot(name, folder=folder, selector=selector)
+ return
self.wait_for_ready_state_complete()
if selector and by:
selector, by = self.__recalculate_selector(selector, by)
@@ -4565,12 +4646,44 @@ def activate_design_mode(self):
script = """document.designMode = 'on';"""
self.execute_script(script)
- def deactivate_design_mode(self):
+ def deactivate_design_mode(self, url=None):
# Deactivate Chrome's Design Mode.
self.wait_for_ready_state_complete()
script = """document.designMode = 'off';"""
self.execute_script(script)
+ def activate_cdp_mode(self, url=None):
+ if hasattr(self.driver, "_is_using_uc") and self.driver._is_using_uc:
+ self.driver.uc_open_with_cdp_mode(url)
+ else:
+ # Fix Chrome-130 issues by creating a user-data-dir in advance
+ if (
+ (
+ not self.user_data_dir
+ or not os.path.exists(self.user_data_dir)
+ )
+ and self.browser == "chrome"
+ ):
+ import tempfile
+ user_data_dir = os.path.normpath(tempfile.mkdtemp())
+ self.user_data_dir = user_data_dir
+ sb_config.user_data_dir = user_data_dir
+ try:
+ driver = self.get_new_driver(
+ user_data_dir=user_data_dir,
+ undetectable=True,
+ headless2=True,
+ )
+ time.sleep(0.555)
+ except Exception:
+ pass
+ finally:
+ with suppress(Exception):
+ driver.quit()
+ self.get_new_driver(undetectable=True)
+ self.driver.uc_open_with_cdp_mode(url)
+ self.cdp = self.driver.cdp
+
def activate_recorder(self):
from seleniumbase.js_code.recorder_js import recorder_js
@@ -5589,6 +5702,9 @@ def bring_active_window_to_front(self):
"""Brings the active browser window to the front (on top).
Useful when multiple drivers are being used at the same time."""
self.__check_scope()
+ if self.__is_cdp_swap_needed():
+ self.cdp.bring_active_window_to_front()
+ return
with suppress(Exception):
if not self.__is_in_frame():
# Only bring the window to the front if not in a frame
@@ -5804,6 +5920,7 @@ def highlight(
scroll - the option to scroll to the element first (Default: True)
timeout - the time to wait for the element to appear """
self.__check_scope()
+ self._check_browser()
self.__skip_if_esc()
if isinstance(selector, WebElement):
self.__highlight_element(selector, loops=loops, scroll=scroll)
@@ -6004,6 +6121,9 @@ def scroll_into_view(self, selector, by="css selector", timeout=None):
timeout = settings.SMALL_TIMEOUT
if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:
timeout = self.__get_new_timeout(timeout)
+ if self.__is_cdp_swap_needed():
+ self.cdp.scroll_into_view(selector)
+ return
element = self.wait_for_element_visible(selector, by, timeout=timeout)
self.execute_script("arguments[0].scrollIntoView();", element)
@@ -6046,6 +6166,9 @@ def js_click(
Can be used to click hidden / invisible elements.
If "all_matches" is False, only the first match is clicked.
If "scroll" is False, won't scroll unless running in Demo Mode."""
+ if self.__is_cdp_swap_needed():
+ self.cdp.click(selector)
+ return
self.wait_for_ready_state_complete()
if not timeout or timeout is True:
timeout = settings.SMALL_TIMEOUT
@@ -6414,6 +6537,9 @@ def show_elements(self, selector, by="css selector"):
def remove_element(self, selector, by="css selector"):
"""Remove the first element on the page that matches the selector."""
self.__check_scope()
+ if self.__is_cdp_swap_needed():
+ self.cdp.remove_element(selector)
+ return
element = None
with suppress(Exception):
self.wait_for_element_visible("body", timeout=1.5)
@@ -6445,6 +6571,9 @@ def remove_element(self, selector, by="css selector"):
def remove_elements(self, selector, by="css selector"):
"""Remove all elements on the page that match the selector."""
self.__check_scope()
+ if self.__is_cdp_swap_needed():
+ self.cdp.remove_elements(selector)
+ return
with suppress(Exception):
self.wait_for_element_visible("body", timeout=1.5)
selector, by = self.__recalculate_selector(selector, by)
@@ -7829,6 +7958,15 @@ def is_online(self):
"""Return True if connected to the Internet."""
return self.execute_script("return navigator.onLine;")
+ def is_connected(self):
+ """
+ Return True if WebDriver is connected to the browser.
+ Note that the stealthy CDP-Driver isn't a WebDriver.
+ In CDP Mode, the CDP-Driver controls the web browser.
+ The CDP-Driver can be connected while WebDriver isn't.
+ """
+ return self.driver.is_connected()
+
def is_chromium(self):
"""Return True if the browser is Chrome or Edge."""
self.__check_scope()
@@ -7944,6 +8082,10 @@ def enter_mfa_code(
self.__check_scope()
if not timeout:
timeout = settings.SMALL_TIMEOUT
+ if self.__is_cdp_swap_needed():
+ mfa_code = self.get_mfa_code(totp_key)
+ self.cdp.type(selector, mfa_code + "\n")
+ return
self.wait_for_element_visible(selector, by=by, timeout=timeout)
if self.recorder_mode and self.__current_url_is_recordable():
if self.get_session_storage_item("pause_recorder") == "no":
@@ -7981,6 +8123,9 @@ def set_value(
if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT:
timeout = self.__get_new_timeout(timeout)
selector, by = self.__recalculate_selector(selector, by, xp_ok=False)
+ if self.__is_cdp_swap_needed():
+ self.cdp.type(selector, text)
+ return
self.wait_for_ready_state_complete()
self.wait_for_element_present(selector, by=by, timeout=timeout)
original_selector = selector
@@ -8696,6 +8841,8 @@ def wait_for_element_visible(
timeout = self.__get_new_timeout(timeout)
original_selector = selector
selector, by = self.__recalculate_selector(selector, by)
+ if self.__is_cdp_swap_needed():
+ return self.cdp.select(selector)
if self.__is_shadow_selector(selector):
return self.__get_shadow_element(selector, timeout)
return page_actions.wait_for_element_visible(
@@ -8751,6 +8898,9 @@ def wait_for_element_not_present(
original_selector=original_selector,
)
+ def select_all(self, selector, by="css selector", limit=0):
+ return self.find_elements(selector, by=by, limit=limit)
+
def assert_link(self, link_text, timeout=None):
"""Same as self.assert_link_text()"""
self.__check_scope()
@@ -8825,6 +8975,7 @@ def block_ads(self):
def _check_browser(self):
"""This method raises an exception if the active window is closed.
(This provides a much cleaner exception message in this situation.)"""
+ page_actions._reconnect_if_disconnected(self.driver)
active_window = None
with suppress(Exception):
active_window = self.driver.current_window_handle # Fails if None
@@ -9109,6 +9260,8 @@ def wait_for_element_present(
timeout = self.__get_new_timeout(timeout)
original_selector = selector
selector, by = self.__recalculate_selector(selector, by)
+ if self.__is_cdp_swap_needed():
+ return self.cdp.select(selector)
if self.__is_shadow_selector(selector):
return self.__wait_for_shadow_element_present(selector, timeout)
return page_actions.wait_for_element_present(
@@ -9129,6 +9282,8 @@ def wait_for_element(self, selector, by="css selector", timeout=None):
timeout = self.__get_new_timeout(timeout)
original_selector = selector
selector, by = self.__recalculate_selector(selector, by)
+ if self.__is_cdp_swap_needed():
+ return self.cdp.select(selector)
if self.recorder_mode and self.__current_url_is_recordable():
if self.get_session_storage_item("pause_recorder") == "no":
if by == By.XPATH:
@@ -9189,6 +9344,9 @@ def assert_element_present(
if isinstance(selector, list):
self.assert_elements_present(selector, by=by, timeout=timeout)
return True
+ if self.__is_cdp_swap_needed():
+ self.cdp.assert_element_present(selector)
+ return True
if self.__is_shadow_selector(selector):
self.__assert_shadow_element_present(selector)
return True
@@ -9263,6 +9421,9 @@ def assert_element(self, selector, by="css selector", timeout=None):
timeout = settings.SMALL_TIMEOUT
if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT:
timeout = self.__get_new_timeout(timeout)
+ if self.__is_cdp_swap_needed():
+ self.cdp.assert_element(selector)
+ return True
if isinstance(selector, list):
self.assert_elements(selector, by=by, timeout=timeout)
return True
@@ -9383,6 +9544,8 @@ def wait_for_text_visible(
timeout = self.__get_new_timeout(timeout)
text = self.__get_type_checked_text(text)
selector, by = self.__recalculate_selector(selector, by)
+ if self.__is_cdp_swap_needed():
+ return self.cdp.find_element(selector)
if self.__is_shadow_selector(selector):
return self.__wait_for_shadow_text_visible(text, selector, timeout)
return page_actions.wait_for_text_visible(
@@ -9551,6 +9714,11 @@ def assert_text(
self.__highlight_with_assert_success(
messenger_post, selector, by
)
+ elif self.__is_cdp_swap_needed():
+ self.cdp.assert_text(text, selector)
+ return True
+ elif not self.is_connected():
+ self.connect()
elif self.__is_shadow_selector(selector):
self.__assert_shadow_text_visible(text, selector, timeout)
return True
@@ -9596,6 +9764,9 @@ def assert_exact_text(
timeout = self.__get_new_timeout(timeout)
original_selector = selector
selector, by = self.__recalculate_selector(selector, by)
+ if self.__is_cdp_swap_needed():
+ self.cdp.assert_exact_text(text, selector)
+ return True
if self.__is_shadow_selector(selector):
self.__assert_exact_shadow_text_visible(text, selector, timeout)
return True
@@ -10578,6 +10749,12 @@ def __get_new_timeout(self, timeout):
############
+ def __is_cdp_swap_needed(self):
+ """If the driver is disconnected, use a CDP method when available."""
+ return shared_utils.is_cdp_swap_needed(self.driver)
+
+ ############
+
def __check_scope(self):
if hasattr(self, "browser"): # self.browser stores the type of browser
return # All good: setUp() already initialized variables in "self"
@@ -14764,6 +14941,31 @@ def setUp(self, masterqa_mode=False):
self.__js_start_time = int(time.time() * 1000.0)
else:
# Launch WebDriver for both pytest and pynose
+
+ # Fix Chrome-130 issues by creating a user-data-dir in advance
+ if (
+ self.undetectable
+ and (
+ not self.user_data_dir
+ or not os.path.exists(self.user_data_dir)
+ )
+ and self.browser == "chrome"
+ ):
+ import tempfile
+ user_data_dir = os.path.normpath(tempfile.mkdtemp())
+ self.user_data_dir = user_data_dir
+ sb_config.user_data_dir = user_data_dir
+ try:
+ driver = self.get_new_driver(
+ user_data_dir=user_data_dir,
+ headless2=True,
+ )
+ time.sleep(0.555)
+ except Exception:
+ pass
+ finally:
+ with suppress(Exception):
+ driver.quit()
self.driver = self.get_new_driver(
browser=self.browser,
headless=self.headless,
diff --git a/seleniumbase/fixtures/constants.py b/seleniumbase/fixtures/constants.py
index 94c8c19068a..145bd5b8cf5 100644
--- a/seleniumbase/fixtures/constants.py
+++ b/seleniumbase/fixtures/constants.py
@@ -375,6 +375,7 @@ class Mobile:
class UC:
RECONNECT_TIME = 2.4 # Seconds
+ CDP_MODE_OPEN_WAIT = 0.9 # Seconds
class ValidBrowsers:
diff --git a/seleniumbase/fixtures/js_utils.py b/seleniumbase/fixtures/js_utils.py
index ff85ed57f32..52bf9154eac 100644
--- a/seleniumbase/fixtures/js_utils.py
+++ b/seleniumbase/fixtures/js_utils.py
@@ -243,6 +243,10 @@ def escape_quotes_if_needed(string):
def is_in_frame(driver):
# Returns True if the driver has switched to a frame.
# Returns False if the driver was on default content.
+ from seleniumbase.fixtures import shared_utils
+
+ if shared_utils.is_cdp_swap_needed(driver):
+ return False
in_basic_frame = driver.execute_script(
"""
var frame = window.frameElement;
diff --git a/seleniumbase/fixtures/page_actions.py b/seleniumbase/fixtures/page_actions.py
index 3e0b1506cd1..b3d9a206f37 100644
--- a/seleniumbase/fixtures/page_actions.py
+++ b/seleniumbase/fixtures/page_actions.py
@@ -49,6 +49,8 @@ def is_element_present(driver, selector, by="css selector"):
@Returns
Boolean (is element present)
"""
+ if __is_cdp_swap_needed(driver):
+ return driver.cdp.is_element_present(selector)
selector, by = page_utils.swap_selector_and_by_if_reversed(selector, by)
try:
driver.find_element(by=by, value=selector)
@@ -67,6 +69,8 @@ def is_element_visible(driver, selector, by="css selector"):
@Returns
Boolean (is element visible)
"""
+ if __is_cdp_swap_needed(driver):
+ return driver.cdp.is_element_visible(selector)
selector, by = page_utils.swap_selector_and_by_if_reversed(selector, by)
try:
element = driver.find_element(by=by, value=selector)
@@ -85,6 +89,7 @@ def is_element_clickable(driver, selector, by="css selector"):
@Returns
Boolean (is element clickable)
"""
+ _reconnect_if_disconnected(driver)
try:
element = driver.find_element(by=by, value=selector)
if element.is_displayed() and element.is_enabled():
@@ -104,6 +109,7 @@ def is_element_enabled(driver, selector, by="css selector"):
@Returns
Boolean (is element enabled)
"""
+ _reconnect_if_disconnected(driver)
try:
element = driver.find_element(by=by, value=selector)
return element.is_enabled()
@@ -122,6 +128,7 @@ def is_text_visible(driver, text, selector="html", by="css selector"):
@Returns
Boolean (is text visible)
"""
+ _reconnect_if_disconnected(driver)
selector, by = page_utils.swap_selector_and_by_if_reversed(selector, by)
text = str(text)
try:
@@ -151,6 +158,7 @@ def is_exact_text_visible(driver, text, selector, by="css selector"):
@Returns
Boolean (is text visible)
"""
+ _reconnect_if_disconnected(driver)
selector, by = page_utils.swap_selector_and_by_if_reversed(selector, by)
text = str(text)
try:
@@ -185,6 +193,7 @@ def is_attribute_present(
@Returns
Boolean (is attribute present)
"""
+ _reconnect_if_disconnected(driver)
try:
element = driver.find_element(by=by, value=selector)
found_value = element.get_attribute(attribute)
@@ -211,6 +220,7 @@ def is_non_empty_text_visible(driver, selector, by="css selector"):
@Returns
Boolean (is any text visible in the element with the selector)
"""
+ _reconnect_if_disconnected(driver)
try:
element = driver.find_element(by=by, value=selector)
element_text = element.text
@@ -235,6 +245,7 @@ def hover_on_element(driver, selector, by="css selector"):
selector - the locator for identifying the page element (required)
by - the type of selector being used (Default: "css selector")
"""
+ _reconnect_if_disconnected(driver)
element = driver.find_element(by=by, value=selector)
hover = ActionChains(driver).move_to_element(element)
hover.perform()
@@ -245,6 +256,7 @@ def hover_element(driver, element):
"""
Similar to hover_on_element(), but uses found element, not a selector.
"""
+ _reconnect_if_disconnected(driver)
hover = ActionChains(driver).move_to_element(element)
hover.perform()
return element
@@ -276,6 +288,7 @@ def hover_and_click(
timeout - number of seconds to wait for click element to appear after hover
js_click - the option to use js_click() instead of click() on the last part
"""
+ _reconnect_if_disconnected(driver)
start_ms = time.time() * 1000.0
stop_ms = start_ms + (timeout * 1000.0)
element = driver.find_element(by=hover_by, value=hover_selector)
@@ -315,6 +328,7 @@ def hover_element_and_click(
"""
Similar to hover_and_click(), but assumes top element is already found.
"""
+ _reconnect_if_disconnected(driver)
start_ms = time.time() * 1000.0
stop_ms = start_ms + (timeout * 1000.0)
hover = ActionChains(driver).move_to_element(element)
@@ -347,6 +361,7 @@ def hover_element_and_double_click(
click_by="css selector",
timeout=settings.SMALL_TIMEOUT,
):
+ _reconnect_if_disconnected(driver)
start_ms = time.time() * 1000.0
stop_ms = start_ms + (timeout * 1000.0)
hover = ActionChains(driver).move_to_element(element)
@@ -398,6 +413,7 @@ def wait_for_element_present(
@Returns
A web element object
"""
+ _reconnect_if_disconnected(driver)
element = None
start_ms = time.time() * 1000.0
stop_ms = start_ms + (timeout * 1000.0)
@@ -457,6 +473,7 @@ def wait_for_element_visible(
@Returns
A web element object
"""
+ _reconnect_if_disconnected(driver)
element = None
is_present = False
start_ms = time.time() * 1000.0
@@ -538,6 +555,7 @@ def wait_for_text_visible(
@Returns
A web element object that contains the text searched for
"""
+ _reconnect_if_disconnected(driver)
element = None
is_present = False
full_text = None
@@ -648,6 +666,7 @@ def wait_for_exact_text_visible(
@Returns
A web element object that contains the text searched for
"""
+ _reconnect_if_disconnected(driver)
element = None
is_present = False
actual_text = None
@@ -758,6 +777,7 @@ def wait_for_attribute(
@Returns
A web element object that contains the expected attribute/value
"""
+ _reconnect_if_disconnected(driver)
element = None
element_present = False
attribute_present = False
@@ -844,6 +864,7 @@ def wait_for_element_clickable(
@Returns
A web element object
"""
+ _reconnect_if_disconnected(driver)
element = None
is_present = False
is_visible = False
@@ -938,6 +959,7 @@ def wait_for_element_absent(
timeout - the time to wait for elements in seconds
original_selector - handle pre-converted ":contains(TEXT)" selector
"""
+ _reconnect_if_disconnected(driver)
start_ms = time.time() * 1000.0
stop_ms = start_ms + (timeout * 1000.0)
for x in range(int(timeout * 10)):
@@ -985,6 +1007,7 @@ def wait_for_element_not_visible(
timeout - the time to wait for the element in seconds
original_selector - handle pre-converted ":contains(TEXT)" selector
"""
+ _reconnect_if_disconnected(driver)
start_ms = time.time() * 1000.0
stop_ms = start_ms + (timeout * 1000.0)
for x in range(int(timeout * 10)):
@@ -1037,6 +1060,7 @@ def wait_for_text_not_visible(
@Returns
A web element object that contains the text searched for
"""
+ _reconnect_if_disconnected(driver)
text = str(text)
start_ms = time.time() * 1000.0
stop_ms = start_ms + (timeout * 1000.0)
@@ -1080,6 +1104,7 @@ def wait_for_exact_text_not_visible(
@Returns
A web element object that contains the text searched for
"""
+ _reconnect_if_disconnected(driver)
text = str(text)
start_ms = time.time() * 1000.0
stop_ms = start_ms + (timeout * 1000.0)
@@ -1122,6 +1147,7 @@ def wait_for_non_empty_text_visible(
@Returns
The web element object that has text
"""
+ _reconnect_if_disconnected(driver)
start_ms = time.time() * 1000.0
stop_ms = start_ms + (timeout * 1000.0)
element = None
@@ -1239,6 +1265,7 @@ def find_visible_elements(driver, selector, by="css selector", limit=0):
by - the type of selector being used (Default: "css selector")
limit - the maximum number of elements to return if > 0.
"""
+ _reconnect_if_disconnected(driver)
elements = driver.find_elements(by=by, value=selector)
if limit and limit > 0 and len(elements) > limit:
elements = elements[:limit]
@@ -1276,6 +1303,7 @@ def save_screenshot(
If the folder provided doesn't exist, it will get created.
The screenshot will be in PNG format: (*.png)
"""
+ _reconnect_if_disconnected(driver)
if not name.endswith(".png"):
name = name + ".png"
if folder:
@@ -1314,6 +1342,7 @@ def save_page_source(driver, name, folder=None):
"""
from seleniumbase.core import log_helper
+ _reconnect_if_disconnected(driver)
if not name.endswith(".html"):
name = name + ".html"
if folder:
@@ -1340,6 +1369,7 @@ def wait_for_and_accept_alert(driver, timeout=settings.LARGE_TIMEOUT):
driver - the webdriver object (required)
timeout - the time to wait for the alert in seconds
"""
+ _reconnect_if_disconnected(driver)
alert = wait_for_and_switch_to_alert(driver, timeout)
alert_text = alert.text
alert.accept()
@@ -1353,6 +1383,7 @@ def wait_for_and_dismiss_alert(driver, timeout=settings.LARGE_TIMEOUT):
driver - the webdriver object (required)
timeout - the time to wait for the alert in seconds
"""
+ _reconnect_if_disconnected(driver)
alert = wait_for_and_switch_to_alert(driver, timeout)
alert_text = alert.text
alert.dismiss()
@@ -1368,6 +1399,7 @@ def wait_for_and_switch_to_alert(driver, timeout=settings.LARGE_TIMEOUT):
driver - the webdriver object (required)
timeout - the time to wait for the alert in seconds
"""
+ _reconnect_if_disconnected(driver)
start_ms = time.time() * 1000.0
stop_ms = start_ms + (timeout * 1000.0)
for x in range(int(timeout * 10)):
@@ -1395,6 +1427,7 @@ def switch_to_frame(driver, frame, timeout=settings.SMALL_TIMEOUT):
frame - the frame element, name, id, index, or selector
timeout - the time to wait for the alert in seconds
"""
+ _reconnect_if_disconnected(driver)
start_ms = time.time() * 1000.0
stop_ms = start_ms + (timeout * 1000.0)
for x in range(int(timeout * 10)):
@@ -1460,6 +1493,7 @@ def switch_to_window(
timeout - the time to wait for the window in seconds
uc_lock - if UC Mode and True, switch_to_window() uses thread-locking
"""
+ _reconnect_if_disconnected(driver)
if window == -1:
window = len(driver.window_handles) - 1
start_ms = time.time() * 1000.0
@@ -1513,11 +1547,36 @@ def switch_to_window(
timeout_exception(Exception, message)
+############
+
+# Special methods for use with UC Mode
+
+def _reconnect_if_disconnected(driver):
+ if (
+ hasattr(driver, "_is_using_uc")
+ and driver._is_using_uc
+ and hasattr(driver, "_is_connected")
+ and not driver._is_connected
+ and hasattr(driver, "is_connected")
+ and not driver.is_connected()
+ ):
+ with suppress(Exception):
+ driver.connect()
+
+
+def __is_cdp_swap_needed(driver):
+ """If the driver is disconnected, use a CDP method when available."""
+ return shared_utils.is_cdp_swap_needed(driver)
+
+
############
# Support methods for direct use from driver
def open_url(driver, url):
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.open(url)
+ return
url = str(url).strip() # Remove leading and trailing whitespace
if not page_utils.looks_like_a_page_url(url):
if page_utils.is_valid_url("https://" + url):
@@ -1527,6 +1586,9 @@ def open_url(driver, url):
def click(driver, selector, by="css selector", timeout=settings.SMALL_TIMEOUT):
selector, by = page_utils.recalculate_selector(selector, by)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.click(selector)
+ return
element = wait_for_element_clickable(
driver, selector, by=by, timeout=timeout
)
@@ -1534,6 +1596,9 @@ def click(driver, selector, by="css selector", timeout=settings.SMALL_TIMEOUT):
def click_link(driver, link_text, timeout=settings.SMALL_TIMEOUT):
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.click_link(link_text)
+ return
element = wait_for_element_clickable(
driver, link_text, by="link text", timeout=timeout
)
@@ -1544,6 +1609,9 @@ def click_if_visible(
driver, selector, by="css selector", timeout=0
):
selector, by = page_utils.recalculate_selector(selector, by)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.click_if_visible(selector)
+ return
if is_element_visible(driver, selector, by=by):
click(driver, selector, by=by, timeout=1)
elif timeout > 0:
@@ -1556,6 +1624,9 @@ def click_if_visible(
def click_active_element(driver):
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.click_active_element()
+ return
driver.execute_script("document.activeElement.click();")
@@ -1563,6 +1634,9 @@ def js_click(
driver, selector, by="css selector", timeout=settings.SMALL_TIMEOUT
):
selector, by = page_utils.recalculate_selector(selector, by)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.click(selector)
+ return
element = wait_for_element_present(
driver, selector, by=by, timeout=timeout
)
@@ -1588,6 +1662,9 @@ def send_keys(
driver, selector, text, by="css selector", timeout=settings.LARGE_TIMEOUT
):
selector, by = page_utils.recalculate_selector(selector, by)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.send_keys(selector, text)
+ return
element = wait_for_element_present(
driver, selector, by=by, timeout=timeout
)
@@ -1602,6 +1679,9 @@ def press_keys(
driver, selector, text, by="css selector", timeout=settings.LARGE_TIMEOUT
):
selector, by = page_utils.recalculate_selector(selector, by)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.press_keys(selector, text)
+ return
element = wait_for_element_present(
driver, selector, by=by, timeout=timeout
)
@@ -1618,6 +1698,9 @@ def update_text(
driver, selector, text, by="css selector", timeout=settings.LARGE_TIMEOUT
):
selector, by = page_utils.recalculate_selector(selector, by)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.type(selector, text)
+ return
element = wait_for_element_clickable(
driver, selector, by=by, timeout=timeout
)
@@ -1631,6 +1714,9 @@ def update_text(
def submit(driver, selector, by="css selector"):
selector, by = page_utils.recalculate_selector(selector, by)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.send_keys(selector, "\r\n")
+ return
element = wait_for_element_clickable(
driver, selector, by=by, timeout=settings.SMALL_TIMEOUT
)
@@ -1655,6 +1741,9 @@ def assert_element_visible(
elif page_utils.is_valid_by(selector):
original_selector = by
selector, by = page_utils.recalculate_selector(selector, by)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.assert_element(selector)
+ return True
wait_for_element_visible(
driver,
selector,
@@ -1673,6 +1762,9 @@ def assert_element_present(
elif page_utils.is_valid_by(selector):
original_selector = by
selector, by = page_utils.recalculate_selector(selector, by)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.assert_element_present(selector)
+ return True
wait_for_element_present(
driver,
selector,
@@ -1708,6 +1800,9 @@ def assert_text(
timeout=settings.SMALL_TIMEOUT,
):
selector, by = page_utils.recalculate_selector(selector, by)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.assert_text(text, selector)
+ return True
wait_for_text_visible(
driver, text.strip(), selector, by=by, timeout=timeout
)
@@ -1721,6 +1816,9 @@ def assert_exact_text(
timeout=settings.SMALL_TIMEOUT,
):
selector, by = page_utils.recalculate_selector(selector, by)
+ if __is_cdp_swap_needed(driver):
+ driver.cdp.assert_exact_text(text, selector)
+ return True
wait_for_exact_text_visible(
driver, text.strip(), selector, by=by, timeout=timeout
)
@@ -1763,6 +1861,8 @@ def wait_for_element(
elif page_utils.is_valid_by(selector):
original_selector = by
selector, by = page_utils.recalculate_selector(selector, by)
+ if __is_cdp_swap_needed(driver):
+ return driver.cdp.select(selector)
return wait_for_element_visible(
driver=driver,
selector=selector,
@@ -1784,6 +1884,8 @@ def wait_for_selector(
elif page_utils.is_valid_by(selector):
original_selector = by
selector, by = page_utils.recalculate_selector(selector, by)
+ if __is_cdp_swap_needed(driver):
+ return driver.cdp.select(selector)
return wait_for_element_present(
driver=driver,
selector=selector,
@@ -1801,6 +1903,8 @@ def wait_for_text(
timeout=settings.LARGE_TIMEOUT,
):
selector, by = page_utils.recalculate_selector(selector, by)
+ if __is_cdp_swap_needed(driver):
+ return driver.cdp.find_element(selector)
return wait_for_text_visible(
driver=driver,
text=text,
@@ -1848,6 +1952,8 @@ def get_text(
by="css selector",
timeout=settings.LARGE_TIMEOUT
):
+ if __is_cdp_swap_needed(driver):
+ return driver.cdp.get_text(selector)
element = wait_for_element(
driver=driver,
selector=selector,
diff --git a/seleniumbase/fixtures/shared_utils.py b/seleniumbase/fixtures/shared_utils.py
index 98092da4c1a..d7e85f77e89 100644
--- a/seleniumbase/fixtures/shared_utils.py
+++ b/seleniumbase/fixtures/shared_utils.py
@@ -73,6 +73,32 @@ def fix_colorama_if_windows():
colorama.just_fix_windows_console()
+def fix_url_as_needed(url):
+ if not url:
+ url = "data:,"
+ elif url.startswith("//"):
+ url = "https:" + url
+ elif ":" not in url:
+ url = "https://" + url
+ return url
+
+
+def is_cdp_swap_needed(driver):
+ """
+ When someone is using CDP Mode with a disconnected webdriver,
+ but they forget to reconnect before calling a webdriver method,
+ this method is used to substitute the webdriver method for a
+ CDP Mode method instead, which keeps CDP Stealth Mode enabled.
+ For other webdriver methods, SeleniumBase will reconnect first.
+ """
+ return (
+ driver.is_cdp_mode_active()
+ # and hasattr(driver, "_is_connected")
+ # and not driver._is_connected
+ and not driver.is_connected()
+ )
+
+
def format_exc(exception, message):
"""Formats an exception message to make the output cleaner."""
from selenium.common.exceptions import ElementNotVisibleException
diff --git a/seleniumbase/plugins/driver_manager.py b/seleniumbase/plugins/driver_manager.py
index 0041da03dd4..64f9250ecd0 100644
--- a/seleniumbase/plugins/driver_manager.py
+++ b/seleniumbase/plugins/driver_manager.py
@@ -124,10 +124,13 @@ def Driver(
d_width=None, # Set device width
d_height=None, # Set device height
d_p_r=None, # Set device pixel ratio
+ position=None, # Shortcut / Duplicate of "window_position".
+ size=None, # Shortcut / Duplicate of "window_size".
uc=None, # Shortcut / Duplicate of "undetectable".
undetected=None, # Shortcut / Duplicate of "undetectable".
uc_cdp=None, # Shortcut / Duplicate of "uc_cdp_events".
uc_sub=None, # Shortcut / Duplicate of "uc_subprocess".
+ locale=None, # Shortcut / Duplicate of "locale_code".
log_cdp=None, # Shortcut / Duplicate of "log_cdp_events".
ad_block=None, # Shortcut / Duplicate of "ad_block_on".
server=None, # Shortcut / Duplicate of "servername".
@@ -216,10 +219,13 @@ def Driver(
d_width (int): Set device width
d_height (int): Set device height
d_p_r (float): Set device pixel ratio
+ position (x,y): Shortcut / Duplicate of "window_position".
+ size (w,h): Shortcut / Duplicate of "window_size".
uc (bool): Shortcut / Duplicate of "undetectable".
undetected (bool): Shortcut / Duplicate of "undetectable".
uc_cdp (bool): Shortcut / Duplicate of "uc_cdp_events".
uc_sub (bool): Shortcut / Duplicate of "uc_subprocess".
+ locale (str): Shortcut / Duplicate of "locale_code".
log_cdp (bool): Shortcut / Duplicate of "log_cdp_events".
ad_block (bool): Shortcut / Duplicate of "ad_block_on".
server (str): Shortcut / Duplicate of "servername".
@@ -433,6 +439,8 @@ def Driver(
break
count += 1
disable_features = d_f
+ if window_position is None and position is not None:
+ window_position = position
w_p = window_position
if w_p is None and "--window-position" in arg_join:
count = 0
@@ -446,29 +454,31 @@ def Driver(
w_p = None
break
count += 1
- window_position = w_p
- if window_position:
- if window_position.count(",") != 1:
- message = (
- '\n\n window_position expects an "x,y" string!'
- '\n (Your input was: "%s")\n' % window_position
- )
- raise Exception(message)
- window_position = window_position.replace(" ", "")
- win_x = None
- win_y = None
- try:
- win_x = int(window_position.split(",")[0])
- win_y = int(window_position.split(",")[1])
- except Exception:
- message = (
- '\n\n Expecting integer values for "x,y"!'
- '\n (window_position input was: "%s")\n'
- % window_position
- )
- raise Exception(message)
- settings.WINDOW_START_X = win_x
- settings.WINDOW_START_Y = win_y
+ window_position = w_p
+ if window_position:
+ if window_position.count(",") != 1:
+ message = (
+ '\n\n window_position expects an "x,y" string!'
+ '\n (Your input was: "%s")\n' % window_position
+ )
+ raise Exception(message)
+ window_position = window_position.replace(" ", "")
+ win_x = None
+ win_y = None
+ try:
+ win_x = int(window_position.split(",")[0])
+ win_y = int(window_position.split(",")[1])
+ except Exception:
+ message = (
+ '\n\n Expecting integer values for "x,y"!'
+ '\n (window_position input was: "%s")\n'
+ % window_position
+ )
+ raise Exception(message)
+ settings.WINDOW_START_X = win_x
+ settings.WINDOW_START_Y = win_y
+ if window_size is None and size is not None:
+ window_size = size
w_s = window_size
if w_s is None and "--window-size" in arg_join:
count = 0
@@ -482,30 +492,30 @@ def Driver(
w_s = None
break
count += 1
- window_size = w_s
- if window_size:
- if window_size.count(",") != 1:
- message = (
- '\n\n window_size expects a "width,height" string!'
- '\n (Your input was: "%s")\n' % window_size
- )
- raise Exception(message)
- window_size = window_size.replace(" ", "")
- width = None
- height = None
- try:
- width = int(window_size.split(",")[0])
- height = int(window_size.split(",")[1])
- except Exception:
- message = (
- '\n\n Expecting integer values for "width,height"!'
- '\n (window_size input was: "%s")\n' % window_size
- )
- raise Exception(message)
- settings.CHROME_START_WIDTH = width
- settings.CHROME_START_HEIGHT = height
- settings.HEADLESS_START_WIDTH = width
- settings.HEADLESS_START_HEIGHT = height
+ window_size = w_s
+ if window_size:
+ if window_size.count(",") != 1:
+ message = (
+ '\n\n window_size expects a "width,height" string!'
+ '\n (Your input was: "%s")\n' % window_size
+ )
+ raise Exception(message)
+ window_size = window_size.replace(" ", "")
+ width = None
+ height = None
+ try:
+ width = int(window_size.split(",")[0])
+ height = int(window_size.split(",")[1])
+ except Exception:
+ message = (
+ '\n\n Expecting integer values for "width,height"!'
+ '\n (window_size input was: "%s")\n' % window_size
+ )
+ raise Exception(message)
+ settings.CHROME_START_WIDTH = width
+ settings.CHROME_START_HEIGHT = height
+ settings.HEADLESS_START_WIDTH = width
+ settings.HEADLESS_START_HEIGHT = height
if agent is None and "--agent" in arg_join:
count = 0
for arg in sys_argv:
@@ -734,6 +744,8 @@ def Driver(
swiftshader = True
else:
swiftshader = False
+ if locale is not None and locale_code is None:
+ locale_code = locale
if ad_block is not None and ad_block_on is None:
ad_block_on = ad_block
if ad_block_on is None:
@@ -779,6 +791,83 @@ def Driver(
# Launch a web browser
from seleniumbase.core import browser_launcher
+ # Fix Chrome-130 issues by creating a user-data-dir in advance
+ if undetectable and not user_data_dir and browser == "chrome":
+ import tempfile
+ import time
+ user_data_dir = (
+ os.path.normpath(tempfile.mkdtemp())
+ )
+ try:
+ decoy_driver = browser_launcher.get_driver(
+ browser_name=browser_name,
+ headless=False,
+ locale_code=locale_code,
+ use_grid=use_grid,
+ protocol=protocol,
+ servername=servername,
+ port=port,
+ proxy_string=proxy_string,
+ proxy_bypass_list=proxy_bypass_list,
+ proxy_pac_url=proxy_pac_url,
+ multi_proxy=multi_proxy,
+ user_agent=user_agent,
+ cap_file=cap_file,
+ cap_string=cap_string,
+ recorder_ext=recorder_ext,
+ disable_cookies=disable_cookies,
+ disable_js=disable_js,
+ disable_csp=disable_csp,
+ enable_ws=enable_ws,
+ enable_sync=enable_sync,
+ use_auto_ext=use_auto_ext,
+ undetectable=undetectable,
+ uc_cdp_events=uc_cdp_events,
+ uc_subprocess=uc_subprocess,
+ log_cdp_events=log_cdp_events,
+ no_sandbox=no_sandbox,
+ disable_gpu=disable_gpu,
+ headless1=False,
+ headless2=True,
+ incognito=incognito,
+ guest_mode=guest_mode,
+ dark_mode=dark_mode,
+ devtools=devtools,
+ remote_debug=remote_debug,
+ enable_3d_apis=enable_3d_apis,
+ swiftshader=swiftshader,
+ ad_block_on=ad_block_on,
+ host_resolver_rules=host_resolver_rules,
+ block_images=block_images,
+ do_not_track=do_not_track,
+ chromium_arg=chromium_arg,
+ firefox_arg=firefox_arg,
+ firefox_pref=firefox_pref,
+ user_data_dir=user_data_dir,
+ extension_zip=extension_zip,
+ extension_dir=extension_dir,
+ disable_features=disable_features,
+ binary_location=binary_location,
+ driver_version=driver_version,
+ page_load_strategy=page_load_strategy,
+ use_wire=use_wire,
+ external_pdf=external_pdf,
+ test_id=test_id,
+ mobile_emulator=is_mobile,
+ device_width=d_width,
+ device_height=d_height,
+ device_pixel_ratio=d_p_r,
+ browser=browser_name,
+ )
+ time.sleep(0.555)
+ except Exception:
+ pass
+ finally:
+ try:
+ decoy_driver.quit()
+ except Exception:
+ pass
+
driver = browser_launcher.get_driver(
browser_name=browser_name,
headless=headless,
diff --git a/seleniumbase/plugins/sb_manager.py b/seleniumbase/plugins/sb_manager.py
index d35e2806ce1..3f00d3f7f84 100644
--- a/seleniumbase/plugins/sb_manager.py
+++ b/seleniumbase/plugins/sb_manager.py
@@ -104,10 +104,13 @@ def SB(
disable_ws=None, # Reverse of "enable_ws". (None and False are different)
disable_beforeunload=None, # Disable the "beforeunload" event on Chromium.
settings_file=None, # A file for overriding default SeleniumBase settings.
+ position=None, # Shortcut / Duplicate of "window_position".
+ size=None, # Shortcut / Duplicate of "window_size".
uc=None, # Shortcut / Duplicate of "undetectable".
undetected=None, # Shortcut / Duplicate of "undetectable".
uc_cdp=None, # Shortcut / Duplicate of "uc_cdp_events".
uc_sub=None, # Shortcut / Duplicate of "uc_subprocess".
+ locale=None, # Shortcut / Duplicate of "locale_code".
log_cdp=None, # Shortcut / Duplicate of "log_cdp_events".
ad_block=None, # Shortcut / Duplicate of "ad_block_on".
server=None, # Shortcut / Duplicate of "servername".
@@ -224,10 +227,13 @@ def SB(
disable_ws (bool): Reverse of "enable_ws". (None and False are different)
disable_beforeunload (bool): Disable the "beforeunload" event on Chromium.
settings_file (str): A file for overriding default SeleniumBase settings.
+ position (x,y): Shortcut / Duplicate of "window_position".
+ size (w,h): Shortcut / Duplicate of "window_size".
uc (bool): Shortcut / Duplicate of "undetectable".
undetected (bool): Shortcut / Duplicate of "undetectable".
uc_cdp (bool): Shortcut / Duplicate of "uc_cdp_events".
uc_sub (bool): Shortcut / Duplicate of "uc_subprocess".
+ locale (str): Shortcut / Duplicate of "locale_code".
log_cdp (bool): Shortcut / Duplicate of "log_cdp_events".
ad_block (bool): Shortcut / Duplicate of "ad_block_on".
server (str): Shortcut / Duplicate of "servername".
@@ -497,8 +503,13 @@ def SB(
break
count += 1
disable_features = d_f
+ if window_position is None and position is not None:
+ window_position = position
w_p = window_position
- if w_p is None and "--window-position" in arg_join:
+ if (
+ w_p is None
+ and ("--window-position" in arg_join or "--position" in arg_join)
+ ):
count = 0
for arg in sys_argv:
if arg.startswith("--window-position="):
@@ -511,6 +522,8 @@ def SB(
break
count += 1
window_position = w_p
+ if window_size is None and size is not None:
+ window_size = size
w_s = window_size
if w_s is None and "--window-size" in arg_join:
count = 0
@@ -890,6 +903,8 @@ def SB(
swiftshader = True
else:
swiftshader = False
+ if locale is not None and locale_code is None:
+ locale_code = locale
if ad_block is not None and ad_block_on is None:
ad_block_on = ad_block
if ad_block_on is None:
@@ -1197,6 +1212,30 @@ def SB(
if not sb_config.multi_proxy:
proxy_helper.remove_proxy_zip_if_present()
start_time = time.time()
+ saved_headless2 = headless2
+
+ # Fix Chrome-130 issues by creating a user-data-dir in advance
+ if undetectable and not user_data_dir and browser == "chrome":
+ import tempfile
+ user_data_dir = (
+ os.path.normpath(tempfile.mkdtemp())
+ )
+ sb.user_data_dir = user_data_dir
+ sb_config.user_data_dir = user_data_dir
+ try:
+ decoy = sb
+ decoy.headless2 = True
+ decoy.setUp()
+ decoy.sleep(0.555)
+ except Exception:
+ pass
+ finally:
+ try:
+ decoy.tearDown()
+ except Exception:
+ pass
+ sb.headless2 = saved_headless2
+
sb.setUp()
test_passed = True # This can change later
teardown_exception = None
diff --git a/seleniumbase/undetected/__init__.py b/seleniumbase/undetected/__init__.py
index 36d6d3a5977..f9b0b44845b 100644
--- a/seleniumbase/undetected/__init__.py
+++ b/seleniumbase/undetected/__init__.py
@@ -441,6 +441,7 @@ def disconnect(self):
if hasattr(self, "service"):
with suppress(Exception):
self.service.stop()
+ self._is_connected = False
time.sleep(0.012)
def connect(self):
@@ -452,6 +453,7 @@ def connect(self):
time.sleep(0.012)
with suppress(Exception):
self.start_session()
+ self._is_connected = True
time.sleep(0.012)
def start_session(self, capabilities=None):
diff --git a/seleniumbase/undetected/cdp_driver/__init__.py b/seleniumbase/undetected/cdp_driver/__init__.py
new file mode 100644
index 00000000000..8cc0fcf4a08
--- /dev/null
+++ b/seleniumbase/undetected/cdp_driver/__init__.py
@@ -0,0 +1 @@
+from seleniumbase.undetected.cdp_driver import cdp_util # noqa
diff --git a/seleniumbase/undetected/cdp_driver/_contradict.py b/seleniumbase/undetected/cdp_driver/_contradict.py
new file mode 100644
index 00000000000..e087e4cf07e
--- /dev/null
+++ b/seleniumbase/undetected/cdp_driver/_contradict.py
@@ -0,0 +1,110 @@
+import warnings as _warnings
+from collections.abc import Mapping as _Mapping, Sequence as _Sequence
+import logging
+
+__logger__ = logging.getLogger(__name__)
+__all__ = ["cdict", "ContraDict"]
+
+
+def cdict(*args, **kwargs):
+ """Factory function"""
+ return ContraDict(*args, **kwargs)
+
+
+class ContraDict(dict):
+ """
+ Directly inherited from dict.
+ Accessible by attribute. o.x == o['x']
+ This works also for all corner cases.
+ Native json.dumps and json.loads work with it.
+
+ Names like "keys", "update", "values" etc won't overwrite the methods,
+ but will just be available using dict lookup notation obj['items']
+ instead of obj.items.
+
+ All key names are converted to snake_case.
+ Hyphen's (-), dot's (.) or whitespaces are replaced by underscore (_).
+ Autocomplete works even if the objects comes from a list.
+ Recursive action. Dict assignments will be converted too.
+ """
+ __module__ = None
+
+ def __init__(self, *args, **kwargs):
+ super().__init__()
+ silent = kwargs.pop("silent", False)
+ _ = dict(*args, **kwargs)
+
+ super().__setattr__("__dict__", self)
+ for k, v in _.items():
+ _check_key(k, self, False, silent)
+ super().__setitem__(k, _wrap(self.__class__, v))
+
+ def __setitem__(self, key, value):
+ super().__setitem__(key, _wrap(self.__class__, value))
+
+ def __setattr__(self, key, value):
+ super().__setitem__(key, _wrap(self.__class__, value))
+
+ def __getattribute__(self, attribute):
+ if attribute in self:
+ return self[attribute]
+ if not _check_key(attribute, self, True, silent=True):
+ return getattr(super(), attribute)
+ return object.__getattribute__(self, attribute)
+
+
+def _wrap(cls, v):
+ if isinstance(v, _Mapping):
+ v = cls(v)
+ elif isinstance(v, _Sequence) and not isinstance(
+ v, (str, bytes, bytearray, set, tuple)
+ ):
+ v = list([_wrap(cls, x) for x in v])
+ return v
+
+
+_warning_names = (
+ "items",
+ "keys",
+ "values",
+ "update",
+ "clear",
+ "copy",
+ "fromkeys",
+ "get",
+ "items",
+ "keys",
+ "pop",
+ "popitem",
+ "setdefault",
+ "update",
+ "values",
+ "class",
+)
+
+_warning_names_message = """\n\
+ While creating a ContraDict object, a key offending key name '{0}'
+ has been found, which might behave unexpected.
+ You will only be able to look it up using key,
+ eg. myobject['{0}']. myobject.{0} will not work with that name."""
+
+
+def _check_key(
+ key: str, mapping: _Mapping, boolean: bool = False, silent=False
+):
+ """Checks `key` and warns if needed.
+ :param key:
+ :param boolean: return True or False instead of passthrough
+ """
+ e = None
+ if not isinstance(key, (str,)):
+ if boolean:
+ return True
+ return key
+ if key.lower() in _warning_names or any(_ in key for _ in ("-", ".")):
+ if not silent:
+ _warnings.warn(_warning_names_message.format(key))
+ e = True
+ if not boolean:
+ return key
+ return not e
diff --git a/seleniumbase/undetected/cdp_driver/browser.py b/seleniumbase/undetected/cdp_driver/browser.py
new file mode 100644
index 00000000000..7571cb87d2b
--- /dev/null
+++ b/seleniumbase/undetected/cdp_driver/browser.py
@@ -0,0 +1,830 @@
+"""CDP-Driver is based on NoDriver"""
+from __future__ import annotations
+import asyncio
+import atexit
+import http.cookiejar
+import json
+import logging
+import os
+import pickle
+import pathlib
+import shutil
+import urllib.parse
+import urllib.request
+import warnings
+from collections import defaultdict
+from typing import List, Set, Tuple, Union
+import mycdp as cdp
+from . import cdp_util as util
+from . import tab
+from ._contradict import ContraDict
+from .config import PathLike, Config, is_posix
+from .connection import Connection
+
+logger = logging.getLogger(__name__)
+
+
+def get_registered_instances():
+ return __registered__instances__
+
+
+def deconstruct_browser():
+ import time
+
+ for _ in __registered__instances__:
+ if not _.stopped:
+ _.stop()
+ for attempt in range(5):
+ try:
+ if _.config and not _.config.uses_custom_data_dir:
+ shutil.rmtree(_.config.user_data_dir, ignore_errors=False)
+ except FileNotFoundError:
+ break
+ except (PermissionError, OSError) as e:
+ if attempt == 4:
+ logger.debug(
+ "Problem removing data dir %s\n"
+ "Consider checking whether it's there "
+ "and remove it by hand\nerror: %s",
+ _.config.user_data_dir,
+ e,
+ )
+ break
+ time.sleep(0.15)
+ continue
+ logging.debug("Temp profile %s was removed." % _.config.user_data_dir)
+
+
+class Browser:
+ """
+ The Browser object is the "root" of the hierarchy
+ and contains a reference to the browser parent process.
+ There should usually be only 1 instance of this.
+ All opened tabs, extra browser screens,
+ and resources will not cause a new Browser process,
+ but rather create additional :class:`Tab` objects.
+ So, besides starting your instance and first/additional tabs,
+ you don't actively use it a lot under normal conditions.
+ Tab objects will represent and control:
+ - tabs (as you know them)
+ - browser windows (new window)
+ - iframe
+ - background processes
+ Note:
+ The Browser object is not instantiated by __init__
+ but using the asynchronous :meth:`Browser.create` method.
+ Note:
+ In Chromium based browsers, there is a parent process which keeps
+ running all the time, even if there are no visible browser windows.
+ Sometimes it's stubborn to close it, so make sure that after using
+ this library, the browser is correctly and fully closed/exited/killed.
+ """
+ _process: asyncio.subprocess.Process
+ _process_pid: int
+ _http: HTTPApi = None
+ _cookies: CookieJar = None
+ config: Config
+ connection: Connection
+
+ @classmethod
+ async def create(
+ cls,
+ config: Config = None,
+ *,
+ user_data_dir: PathLike = None,
+ headless: bool = False,
+ incognito: bool = False,
+ guest: bool = False,
+ browser_executable_path: PathLike = None,
+ browser_args: List[str] = None,
+ sandbox: bool = True,
+ host: str = None,
+ port: int = None,
+ **kwargs,
+ ) -> Browser:
+ """Entry point for creating an instance."""
+ if not config:
+ config = Config(
+ user_data_dir=user_data_dir,
+ headless=headless,
+ incognito=incognito,
+ guest=guest,
+ browser_executable_path=browser_executable_path,
+ browser_args=browser_args or [],
+ sandbox=sandbox,
+ host=host,
+ port=port,
+ **kwargs,
+ )
+ instance = cls(config)
+ await instance.start()
+ return instance
+
+ def __init__(self, config: Config, **kwargs):
+ """
+ Constructor. To create a instance, use :py:meth:`Browser.create(...)`
+ :param config:
+ """
+ try:
+ asyncio.get_running_loop()
+ except RuntimeError:
+ raise RuntimeError(
+ "{0} objects of this class are created "
+ "using await {0}.create()".format(
+ self.__class__.__name__
+ )
+ )
+ self.config = config
+ self.targets: List = []
+ self.info = None
+ self._target = None
+ self._process = None
+ self._process_pid = None
+ self._keep_user_data_dir = None
+ self._is_updating = asyncio.Event()
+ self.connection: Connection = None
+ logger.debug("Session object initialized: %s" % vars(self))
+
+ @property
+ def websocket_url(self):
+ return self.info.webSocketDebuggerUrl
+
+ @property
+ def main_tab(self) -> tab.Tab:
+ """Returns the target which was launched with the browser."""
+ return sorted(
+ self.targets, key=lambda x: x.type_ == "page", reverse=True
+ )[0]
+
+ @property
+ def tabs(self) -> List[tab.Tab]:
+ """Returns the current targets which are of type "page"."""
+ tabs = filter(lambda item: item.type_ == "page", self.targets)
+ return list(tabs)
+
+ @property
+ def cookies(self) -> CookieJar:
+ if not self._cookies:
+ self._cookies = CookieJar(self)
+ return self._cookies
+
+ @property
+ def stopped(self):
+ if self._process and self._process.returncode is None:
+ return False
+ return True
+ # return (self._process and self._process.returncode) or False
+
+ async def wait(self, time: Union[float, int] = 1) -> Browser:
+ """Wait for seconds. Important to use,
+ especially in between page navigation.
+ :param time:
+ """
+ return await asyncio.sleep(time, result=self)
+
+ sleep = wait
+ """Alias for wait"""
+ def _handle_target_update(
+ self,
+ event: Union[
+ cdp.target.TargetInfoChanged,
+ cdp.target.TargetDestroyed,
+ cdp.target.TargetCreated,
+ cdp.target.TargetCrashed,
+ ],
+ ):
+ """This is an internal handler which updates the targets
+ when Chrome emits the corresponding event."""
+ if isinstance(event, cdp.target.TargetInfoChanged):
+ target_info = event.target_info
+ current_tab = next(
+ filter(
+ lambda item: item.target_id == target_info.target_id, self.targets # noqa
+ )
+ )
+ current_target = current_tab.target
+ if logger.getEffectiveLevel() <= 10:
+ changes = util.compare_target_info(
+ current_target, target_info
+ )
+ changes_string = ""
+ for change in changes:
+ key, old, new = change
+ changes_string += f"\n{key}: {old} => {new}\n"
+ logger.debug(
+ "Target #%d has changed: %s"
+ % (self.targets.index(current_tab), changes_string)
+ )
+ current_tab.target = target_info
+ elif isinstance(event, cdp.target.TargetCreated):
+ target_info: cdp.target.TargetInfo = event.target_info
+ from .tab import Tab
+
+ new_target = Tab(
+ (
+ f"ws://{self.config.host}:{self.config.port}"
+ f"/devtools/{target_info.type_ or 'page'}"
+ f"/{target_info.target_id}"
+ ),
+ target=target_info,
+ browser=self,
+ )
+ self.targets.append(new_target)
+ logger.debug(
+ "Target #%d created => %s", len(self.targets), new_target
+ )
+ elif isinstance(event, cdp.target.TargetDestroyed):
+ current_tab = next(
+ filter(
+ lambda item: item.target_id == event.target_id,
+ self.targets,
+ )
+ )
+ logger.debug(
+ "Target removed. id # %d => %s"
+ % (self.targets.index(current_tab), current_tab)
+ )
+ self.targets.remove(current_tab)
+
+ async def get(
+ self,
+ url="chrome://welcome",
+ new_tab: bool = False,
+ new_window: bool = False,
+ ) -> tab.Tab:
+ """Top level get. Utilizes the first tab to retrieve given url.
+ Convenience function known from selenium.
+ This function detects when DOM events have fired during navigation.
+ :param url: The URL to navigate to
+ :param new_tab: Open new tab
+ :param new_window: Open new window
+ :return: Page
+ """
+ if new_tab or new_window:
+ # Create new target using the browser session.
+ target_id = await self.connection.send(
+ cdp.target.create_target(
+ url, new_window=new_window, enable_begin_frame_control=True
+ )
+ )
+ connection: tab.Tab = next(
+ filter(
+ lambda item: item.type_ == "page" and item.target_id == target_id, # noqa
+ self.targets,
+ )
+ )
+ connection.browser = self
+ else:
+ # First tab from browser.tabs
+ connection: tab.Tab = next(
+ filter(lambda item: item.type_ == "page", self.targets)
+ )
+ # Use the tab to navigate to new url
+ frame_id, loader_id, *_ = await connection.send(
+ cdp.page.navigate(url)
+ )
+ # Update the frame_id on the tab
+ connection.frame_id = frame_id
+ connection.browser = self
+ await connection.sleep(0.25)
+ return connection
+
+ async def start(self=None) -> Browser:
+ """Launches the actual browser."""
+ if not self:
+ warnings.warn(
+ "Use ``await Browser.create()`` to create a new instance!"
+ )
+ return
+ if self._process or self._process_pid:
+ if self._process.returncode is not None:
+ return await self.create(config=self.config)
+ warnings.warn(
+ "Ignored! This call has no effect when already running!"
+ )
+ return
+ # self.config.update(kwargs)
+ connect_existing = False
+ if self.config.host is not None and self.config.port is not None:
+ connect_existing = True
+ else:
+ self.config.host = "127.0.0.1"
+ self.config.port = util.free_port()
+ if not connect_existing:
+ logger.debug(
+ "BROWSER EXECUTABLE PATH: %s",
+ self.config.browser_executable_path,
+ )
+ if not pathlib.Path(self.config.browser_executable_path).exists():
+ raise FileNotFoundError(
+ (
+ """
+ ---------------------------------------
+ Could not determine browser executable.
+ ---------------------------------------
+ Browser must be installed in the default location / path!
+ If you are sure about the browser executable,
+ set it using `browser_executable_path='{}` parameter."""
+ ).format(
+ "/path/to/browser/executable"
+ if is_posix
+ else "c:/path/to/your/browser.exe"
+ )
+ )
+ if getattr(self.config, "_extensions", None): # noqa
+ self.config.add_argument(
+ "--load-extension=%s"
+ % ",".join(str(_) for _ in self.config._extensions)
+ ) # noqa
+ exe = self.config.browser_executable_path
+ params = self.config()
+ logger.info(
+ "Starting\n\texecutable :%s\n\narguments:\n%s",
+ exe,
+ "\n\t".join(params),
+ )
+ if not connect_existing:
+ self._process: asyncio.subprocess.Process = (
+ await asyncio.create_subprocess_exec(
+ # self.config.browser_executable_path,
+ # *cmdparams,
+ exe,
+ *params,
+ stdin=asyncio.subprocess.PIPE,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ close_fds=is_posix,
+ )
+ )
+ self._process_pid = self._process.pid
+ self._http = HTTPApi((self.config.host, self.config.port))
+ get_registered_instances().add(self)
+ await asyncio.sleep(0.25)
+ for _ in range(5):
+ try:
+ self.info = ContraDict(
+ await self._http.get("version"), silent=True
+ )
+ except (Exception,):
+ if _ == 4:
+ logger.debug("Could not start", exc_info=True)
+ await self.sleep(0.5)
+ else:
+ break
+ if not self.info:
+ raise Exception(
+ (
+ """
+ --------------------------------
+ Failed to connect to the browser
+ --------------------------------
+ Possibly because you are running as "root".
+ If so, you may need to use no_sandbox=True.
+ """
+ )
+ )
+ self.connection = Connection(
+ self.info.webSocketDebuggerUrl, _owner=self
+ )
+ if self.config.autodiscover_targets:
+ logger.info("Enabling autodiscover targets")
+ self.connection.handlers[cdp.target.TargetInfoChanged] = [
+ self._handle_target_update
+ ]
+ self.connection.handlers[cdp.target.TargetCreated] = [
+ self._handle_target_update
+ ]
+ self.connection.handlers[cdp.target.TargetDestroyed] = [
+ self._handle_target_update
+ ]
+ self.connection.handlers[cdp.target.TargetCrashed] = [
+ self._handle_target_update
+ ]
+ await self.connection.send(
+ cdp.target.set_discover_targets(discover=True)
+ )
+ await self
+ # self.connection.handlers[cdp.inspector.Detached] = [self.stop]
+ # return self
+
+ async def grant_all_permissions(self):
+ """
+ Grant permissions for:
+ accessibilityEvents
+ audioCapture
+ backgroundSync
+ backgroundFetch
+ clipboardReadWrite
+ clipboardSanitizedWrite
+ displayCapture
+ durableStorage
+ geolocation
+ idleDetection
+ localFonts
+ midi
+ midiSysex
+ nfc
+ notifications
+ paymentHandler
+ periodicBackgroundSync
+ protectedMediaIdentifier
+ sensors
+ storageAccess
+ topLevelStorageAccess
+ videoCapture
+ videoCapturePanTiltZoom
+ wakeLockScreen
+ wakeLockSystem
+ windowManagement
+ """
+ permissions = list(cdp.browser.PermissionType)
+ permissions.remove(cdp.browser.PermissionType.FLASH)
+ permissions.remove(cdp.browser.PermissionType.CAPTURED_SURFACE_CONTROL)
+ await self.connection.send(cdp.browser.grant_permissions(permissions))
+
+ async def tile_windows(self, windows=None, max_columns: int = 0):
+ import math
+ try:
+ import mss
+ except Exception:
+ from seleniumbase.fixtures import shared_utils
+ shared_utils.pip_install("mss")
+ import mss
+ m = mss.mss()
+ screen, screen_width, screen_height = 3 * (None,)
+ if m.monitors and len(m.monitors) >= 1:
+ screen = m.monitors[0]
+ screen_width = screen["width"]
+ screen_height = screen["height"]
+ if not screen or not screen_width or not screen_height:
+ warnings.warn("No monitors detected!")
+ return
+ await self
+ distinct_windows = defaultdict(list)
+ if windows:
+ tabs = windows
+ else:
+ tabs = self.tabs
+ for _tab in tabs:
+ window_id, bounds = await _tab.get_window()
+ distinct_windows[window_id].append(_tab)
+ num_windows = len(distinct_windows)
+ req_cols = max_columns or int(num_windows * (19 / 6))
+ req_rows = int(num_windows / req_cols)
+ while req_cols * req_rows < num_windows:
+ req_rows += 1
+ box_w = math.floor((screen_width / req_cols) - 1)
+ box_h = math.floor(screen_height / req_rows)
+ distinct_windows_iter = iter(distinct_windows.values())
+ grid = []
+ for x in range(req_cols):
+ for y in range(req_rows):
+ try:
+ tabs = next(distinct_windows_iter)
+ except StopIteration:
+ continue
+ if not tabs:
+ continue
+ tab = tabs[0]
+ try:
+ pos = [x * box_w, y * box_h, box_w, box_h]
+ grid.append(pos)
+ await tab.set_window_size(*pos)
+ except Exception:
+ logger.info(
+ "Could not set window size. Exception => ",
+ exc_info=True,
+ )
+ continue
+ return grid
+
+ async def _get_targets(self) -> List[cdp.target.TargetInfo]:
+ info = await self.connection.send(
+ cdp.target.get_targets(), _is_update=True
+ )
+ return info
+
+ async def update_targets(self):
+ targets: List[cdp.target.TargetInfo]
+ targets = await self._get_targets()
+ for t in targets:
+ for existing_tab in self.targets:
+ existing_target = existing_tab.target
+ if existing_target.target_id == t.target_id:
+ existing_tab.target.__dict__.update(t.__dict__)
+ break
+ else:
+ self.targets.append(
+ Connection(
+ (
+ f"ws://{self.config.host}:{self.config.port}"
+ f"/devtools/page" # All types are "page"
+ f"/{t.target_id}"
+ ),
+ target=t,
+ _owner=self,
+ )
+ )
+ await asyncio.sleep(0)
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
+ if exc_type and exc_val:
+ raise exc_type(exc_val)
+
+ def __iter__(self):
+ self._i = self.tabs.index(self.main_tab)
+ return self
+
+ def __reversed__(self):
+ return reversed(list(self.tabs))
+
+ def __next__(self):
+ try:
+ return self.tabs[self._i]
+ except IndexError:
+ del self._i
+ raise StopIteration
+ except AttributeError:
+ del self._i
+ raise StopIteration
+ finally:
+ if hasattr(self, "_i"):
+ if self._i != len(self.tabs):
+ self._i += 1
+ else:
+ del self._i
+
+ def stop(self):
+ try:
+ # asyncio.get_running_loop().create_task(
+ # self.connection.send(cdp.browser.close())
+ # )
+ asyncio.get_event_loop().create_task(self.connection.aclose())
+ logger.debug(
+ "Closed the connection using get_event_loop().create_task()"
+ )
+ except RuntimeError:
+ if self.connection:
+ try:
+ # asyncio.run(self.connection.send(cdp.browser.close()))
+ asyncio.run(self.connection.aclose())
+ logger.debug("Closed the connection using asyncio.run()")
+ except Exception:
+ pass
+ for _ in range(3):
+ try:
+ self._process.terminate()
+ logger.info(
+ "Terminated browser with pid %d successfully."
+ % self._process.pid
+ )
+ break
+ except (Exception,):
+ try:
+ self._process.kill()
+ logger.info(
+ "Killed browser with pid %d successfully."
+ % self._process.pid
+ )
+ break
+ except (Exception,):
+ try:
+ if hasattr(self, "browser_process_pid"):
+ os.kill(self._process_pid, 15)
+ logger.info(
+ "Killed browser with pid %d "
+ "using signal 15 successfully."
+ % self._process.pid
+ )
+ break
+ except (TypeError,):
+ logger.info("typerror", exc_info=True)
+ pass
+ except (PermissionError,):
+ logger.info(
+ "Browser already stopped, "
+ "or no permission to kill. Skip."
+ )
+ pass
+ except (ProcessLookupError,):
+ logger.info("Process lookup failure!")
+ pass
+ except (Exception,):
+ raise
+ self._process = None
+ self._process_pid = None
+
+ def __await__(self):
+ # return ( asyncio.sleep(0)).__await__()
+ return self.update_targets().__await__()
+
+ def __del__(self):
+ pass
+
+
+__registered__instances__: Set[Browser] = set()
+
+
+class CookieJar:
+ def __init__(self, browser: Browser):
+ self._browser = browser
+
+ async def get_all(
+ self, requests_cookie_format: bool = False
+ ) -> List[Union[cdp.network.Cookie, "http.cookiejar.Cookie"]]:
+ """
+ Get all cookies.
+ :param requests_cookie_format: when True,
+ returns python http.cookiejar.Cookie objects,
+ compatible with requests library and many others.
+ :type requests_cookie_format: bool
+ """
+ connection = None
+ for _tab in self._browser.tabs:
+ if _tab.closed:
+ continue
+ connection = _tab
+ break
+ else:
+ connection = self._browser.connection
+ cookies = await connection.send(cdp.storage.get_cookies())
+ if requests_cookie_format:
+ import requests.cookies
+
+ return [
+ requests.cookies.create_cookie(
+ name=c.name,
+ value=c.value,
+ domain=c.domain,
+ path=c.path,
+ expires=c.expires,
+ secure=c.secure,
+ )
+ for c in cookies
+ ]
+ return cookies
+
+ async def set_all(self, cookies: List[cdp.network.CookieParam]):
+ """
+ Set cookies.
+ :param cookies: List of cookies
+ """
+ connection = None
+ for _tab in self._browser.tabs:
+ if _tab.closed:
+ continue
+ connection = _tab
+ break
+ else:
+ connection = self._browser.connection
+ cookies = await connection.send(cdp.storage.get_cookies())
+ await connection.send(cdp.storage.set_cookies(cookies))
+
+ async def save(self, file: PathLike = ".session.dat", pattern: str = ".*"):
+ """
+ Save all cookies (or a subset, controlled by `pattern`)
+ to a file to be restored later.
+ :param file:
+ :param pattern: regex style pattern string.
+ any cookie that has a domain, key or value field
+ which matches the pattern will be included.
+ default = ".*" (all)
+ Eg: the pattern "(cf|.com|nowsecure)" will include cookies which:
+ - Have a string "cf" (cloudflare)
+ - Have ".com" in them, in either domain, key or value field.
+ - Contain "nowsecure"
+ :type pattern: str
+ """
+ import re
+
+ pattern = re.compile(pattern)
+ save_path = pathlib.Path(file).resolve()
+ connection = None
+ for _tab in self._browser.tabs:
+ if _tab.closed:
+ continue
+ connection = _tab
+ break
+ else:
+ connection = self._browser.connection
+ cookies = await connection.send(cdp.storage.get_cookies())
+ # if not connection:
+ # return
+ # if not connection.websocket:
+ # return
+ # if connection.websocket.closed:
+ # return
+ cookies = await self.get_all(requests_cookie_format=False)
+ included_cookies = []
+ for cookie in cookies:
+ for match in pattern.finditer(str(cookie.__dict__)):
+ logger.debug(
+ "Saved cookie for matching pattern '%s' => (%s: %s)",
+ pattern.pattern,
+ cookie.name,
+ cookie.value,
+ )
+ included_cookies.append(cookie)
+ break
+ pickle.dump(cookies, save_path.open("w+b"))
+
+ async def load(self, file: PathLike = ".session.dat", pattern: str = ".*"):
+ """
+ Load all cookies (or a subset, controlled by `pattern`)
+ from a file created by :py:meth:`~save_cookies`.
+ :param file:
+ :param pattern: Regex style pattern string.
+ Any cookie that has a domain, key,
+ or value field which matches the pattern will be included.
+ Default = ".*" (all)
+ Eg: the pattern "(cf|.com|nowsecure)" will include cookies which:
+ - Have a string "cf" (cloudflare)
+ - Have ".com" in them, in either domain, key or value field.
+ - Contain "nowsecure"
+ :type pattern: str
+ """
+ import re
+
+ pattern = re.compile(pattern)
+ save_path = pathlib.Path(file).resolve()
+ cookies = pickle.load(save_path.open("r+b"))
+ included_cookies = []
+ connection = None
+ for _tab in self._browser.tabs:
+ if _tab.closed:
+ continue
+ connection = _tab
+ break
+ else:
+ connection = self._browser.connection
+ for cookie in cookies:
+ for match in pattern.finditer(str(cookie.__dict__)):
+ included_cookies.append(cookie)
+ logger.debug(
+ "Loaded cookie for matching pattern '%s' => (%s: %s)",
+ pattern.pattern,
+ cookie.name,
+ cookie.value,
+ )
+ break
+ await connection.send(cdp.storage.set_cookies(included_cookies))
+
+ async def clear(self):
+ """
+ Clear current cookies.
+ Note: This includes all open tabs/windows for this browser.
+ """
+ connection = None
+ for _tab in self._browser.tabs:
+ if _tab.closed:
+ continue
+ connection = _tab
+ break
+ else:
+ connection = self._browser.connection
+ cookies = await connection.send(cdp.storage.get_cookies())
+ if cookies:
+ await connection.send(cdp.storage.clear_cookies())
+
+
+class HTTPApi:
+ def __init__(self, addr: Tuple[str, int]):
+ self.host, self.port = addr
+ self.api = "http://%s:%d" % (self.host, self.port)
+
+ @classmethod
+ def from_target(cls, target):
+ ws_url = urllib.parse.urlparse(target.websocket_url)
+ inst = cls((ws_url.hostname, ws_url.port))
+ return inst
+
+ async def get(self, endpoint: str):
+ return await self._request(endpoint)
+
+ async def post(self, endpoint, data):
+ return await self._request(endpoint, data)
+
+ async def _request(self, endpoint, method: str = "get", data: dict = None):
+ url = urllib.parse.urljoin(
+ self.api, f"json/{endpoint}" if endpoint else "/json"
+ )
+ if data and method.lower() == "get":
+ raise ValueError("get requests cannot contain data")
+ if not url:
+ url = self.api + endpoint
+ request = urllib.request.Request(url)
+ request.method = method
+ request.data = None
+ if data:
+ request.data = json.dumps(data).encode("utf-8")
+
+ response = await asyncio.get_running_loop().run_in_executor(
+ None, lambda: urllib.request.urlopen(request, timeout=10)
+ )
+ return json.loads(response.read())
+
+
+atexit.register(deconstruct_browser)
diff --git a/seleniumbase/undetected/cdp_driver/cdp_util.py b/seleniumbase/undetected/cdp_driver/cdp_util.py
new file mode 100644
index 00000000000..98f80767714
--- /dev/null
+++ b/seleniumbase/undetected/cdp_driver/cdp_util.py
@@ -0,0 +1,301 @@
+"""CDP-Driver is based on NoDriver"""
+from __future__ import annotations
+import asyncio
+import logging
+import time
+import types
+import typing
+from typing import Optional, List, Union, Callable
+from .element import Element
+from .browser import Browser
+from .browser import PathLike
+from .config import Config
+from .tab import Tab
+import mycdp as cdp
+
+logger = logging.getLogger(__name__)
+T = typing.TypeVar("T")
+
+
+async def start(
+ config: Optional[Config] = None,
+ *,
+ user_data_dir: Optional[PathLike] = None,
+ headless: Optional[bool] = False,
+ incognito: Optional[bool] = False,
+ guest: Optional[bool] = False,
+ browser_executable_path: Optional[PathLike] = None,
+ browser_args: Optional[List[str]] = None,
+ sandbox: Optional[bool] = True,
+ lang: Optional[str] = None,
+ host: Optional[str] = None,
+ port: Optional[int] = None,
+ expert: Optional[bool] = None,
+ **kwargs: Optional[dict],
+) -> Browser:
+ """
+ Helper function to launch a browser. It accepts several keyword parameters.
+ Conveniently, you can just call it bare (no parameters) to quickly launch
+ an instance with best practice defaults.
+ Note: New args are expected: Use kwargs only!
+ Note: This should be called ``await start()``
+ :param user_data_dir:
+ :type user_data_dir: PathLike
+ :param headless:
+ :type headless: bool
+ :param browser_executable_path:
+ :type browser_executable_path: PathLike
+ :param browser_args:
+ ["--some-chromeparam=somevalue", "some-other-param=someval"]
+ :type browser_args: List[str]
+ :param sandbox: Default True, but when set to False it adds --no-sandbox
+ to the params, also when using linux under a root user,
+ it adds False automatically (else Chrome won't start).
+ :type sandbox: bool
+ :param lang: language string
+ :type lang: str
+ :param port: If you connect to an existing debuggable session,
+ you can specify the port here.
+ If both host and port are provided,
+ then a local Chrome browser will not be started!
+ :type port: int
+ :param host: If you connect to an existing debuggable session,
+ you can specify the host here.
+ If both host and port are provided,
+ then a local Chrome browser will not be started!
+ :type host: str
+ :param expert: When set to True, "expert" mode is enabled.
+ This means adding: --disable-web-security --disable-site-isolation-trials,
+ as well as some scripts and patching useful for debugging.
+ (For example, ensuring shadow-root is always in "open" mode.)
+ :type expert: bool
+ """
+ if not config:
+ config = Config(
+ user_data_dir,
+ headless,
+ incognito,
+ guest,
+ browser_executable_path,
+ browser_args,
+ sandbox,
+ lang,
+ host=host,
+ port=port,
+ expert=expert,
+ **kwargs,
+ )
+ return await Browser.create(config)
+
+
+def start_sync(*args, **kwargs) -> Browser:
+ loop = asyncio.get_event_loop()
+ headless = False
+ if "headless" in kwargs:
+ headless = kwargs["headless"]
+ decoy_args = kwargs
+ decoy_args["headless"] = True
+ driver = loop.run_until_complete(start(**decoy_args))
+ kwargs["headless"] = headless
+ kwargs["user_data_dir"] = driver.config.user_data_dir
+ driver.stop() # Due to Chrome-130, must stop & start
+ time.sleep(0.15)
+ return loop.run_until_complete(start(*args, **kwargs))
+
+
+async def create_from_driver(driver) -> Browser:
+ """Create a Browser instance from a running driver instance."""
+ from .config import Config
+
+ conf = Config()
+ host, port = driver.options.debugger_address.split(":")
+ conf.host, conf.port = host, int(port)
+ # Create Browser instance
+ browser = await start(conf)
+ browser._process_pid = driver.browser_pid
+ # Stop chromedriver binary
+ driver.service.stop()
+ driver.browser_pid = -1
+ driver.user_data_dir = None
+ return browser
+
+
+def free_port() -> int:
+ """Determines a free port using sockets."""
+ import socket
+
+ free_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ free_socket.bind(("127.0.0.1", 0))
+ free_socket.listen(5)
+ port: int = free_socket.getsockname()[1]
+ free_socket.close()
+ return port
+
+
+def filter_recurse_all(
+ doc: T, predicate: Callable[[cdp.dom.Node, Element], bool]
+) -> List[T]:
+ """
+ Test each child using predicate(child),
+ and return all children for which predicate(child) == True
+ :param doc: The cdp.dom.Node object or :py:class:`cdp_driver.Element`
+ :param predicate: A function which takes a node as first parameter
+ and returns a boolean, where True means include.
+ """
+ if not hasattr(doc, "children"):
+ raise TypeError("Object should have a .children attribute!")
+ out = []
+ if doc and doc.children:
+ for child in doc.children:
+ if predicate(child):
+ out.append(child)
+ if child.shadow_roots is not None:
+ out.extend(
+ filter_recurse_all(child.shadow_roots[0], predicate)
+ )
+ out.extend(filter_recurse_all(child, predicate))
+ return out
+
+
+def filter_recurse(
+ doc: T, predicate: Callable[[cdp.dom.Node, Element], bool]
+) -> T:
+ """
+ Test each child using predicate(child),
+ and return the first child of which predicate(child) == True
+ :param doc: the cdp.dom.Node object or :py:class:`cdp_driver.Element`
+ :param predicate: a function which takes a node as first parameter
+ and returns a boolean, where True means include.
+ """
+ if not hasattr(doc, "children"):
+ raise TypeError("Object should have a .children attribute!")
+ if doc and doc.children:
+ for child in doc.children:
+ if predicate(child):
+ return child
+ if child.shadow_roots:
+ shadow_root_result = filter_recurse(
+ child.shadow_roots[0], predicate
+ )
+ if shadow_root_result:
+ return shadow_root_result
+ result = filter_recurse(child, predicate)
+ if result:
+ return result
+
+
+def circle(
+ x, y=None, radius=10, num=10, dir=0
+) -> typing.Generator[typing.Tuple[float, float], None, None]:
+ """
+ A generator will calculate coordinates around a circle.
+ :param x: start x position
+ :type x: int
+ :param y: start y position
+ :type y: int
+ :param radius: size of the circle
+ :type radius: int
+ :param num: the amount of points calculated
+ (higher => slower, more cpu, but more detailed)
+ :type num: int
+ """
+ import math
+
+ r = radius
+ w = num
+ if not y:
+ y = x
+ a = int(x - r * 2)
+ b = int(y - r * 2)
+ m = (2 * math.pi) / w
+ if dir == 0:
+ # Regular direction
+ ran = 0, w + 1, 1
+ else:
+ # Opposite direction
+ ran = w + 1, 0, -1
+ for i in range(*ran):
+ x = a + r * math.sin(m * i)
+ y = b + r * math.cos(m * i)
+ yield x, y
+
+
+def remove_from_tree(tree: cdp.dom.Node, node: cdp.dom.Node) -> cdp.dom.Node:
+ if not hasattr(tree, "children"):
+ raise TypeError("Object should have a .children attribute!")
+ if tree and tree.children:
+ for child in tree.children:
+ if child.backend_node_id == node.backend_node_id:
+ tree.children.remove(child)
+ remove_from_tree(child, node)
+ return tree
+
+
+async def html_from_tree(
+ tree: Union[cdp.dom.Node, Element], target: Tab
+):
+ if not hasattr(tree, "children"):
+ raise TypeError("Object should have a .children attribute!")
+ out = ""
+ if tree and tree.children:
+ for child in tree.children:
+ if isinstance(child, Element):
+ out += await child.get_html()
+ else:
+ out += await target.send(
+ cdp.dom.get_outer_html(
+ backend_node_id=child.backend_node_id
+ )
+ )
+ out += await html_from_tree(child, target)
+ return out
+
+
+def compare_target_info(
+ info1: cdp.target.TargetInfo, info2: cdp.target.TargetInfo
+) -> List[typing.Tuple[str, typing.Any, typing.Any]]:
+ """
+ When logging mode is set to debug, browser object will log when target info
+ is changed. To provide more meaningful log messages,
+ this function is called to check what has actually changed
+ between the 2 (by simple dict comparison).
+ It returns a list of tuples
+ [ ... ( key_which_has_changed, old_value, new_value) ]
+ :param info1:
+ :param info2:
+ """
+ d1 = info1.__dict__
+ d2 = info2.__dict__
+ return [(k, v, d2[k]) for (k, v) in d1.items() if d2[k] != v]
+
+
+def loop():
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ return loop
+
+
+def cdp_get_module(domain: Union[str, types.ModuleType]):
+ """
+ Get cdp module by given string.
+ :param domain:
+ """
+ import importlib
+
+ if isinstance(domain, types.ModuleType):
+ domain_mod = domain
+ else:
+ try:
+ if domain in ("input",):
+ domain = "input_"
+ domain_mod = getattr(cdp, domain)
+ if not domain_mod:
+ raise AttributeError
+ except AttributeError:
+ try:
+ domain_mod = importlib.import_module(domain)
+ except ModuleNotFoundError:
+ raise ModuleNotFoundError(
+ "Could not find cdp module from input '%s'" % domain
+ )
+ return domain_mod
diff --git a/seleniumbase/undetected/cdp_driver/config.py b/seleniumbase/undetected/cdp_driver/config.py
new file mode 100644
index 00000000000..37c86892970
--- /dev/null
+++ b/seleniumbase/undetected/cdp_driver/config.py
@@ -0,0 +1,322 @@
+import logging
+import os
+import pathlib
+import secrets
+import sys
+import tempfile
+import zipfile
+from typing import Union, List, Optional
+
+__all__ = [
+ "Config",
+ "find_chrome_executable",
+ "temp_profile_dir",
+ "is_root",
+ "is_posix",
+ "PathLike",
+]
+
+logger = logging.getLogger(__name__)
+is_posix = sys.platform.startswith(("darwin", "cygwin", "linux", "linux2"))
+
+PathLike = Union[str, pathlib.Path]
+AUTO = None
+
+
+class Config:
+ """Config object"""
+
+ def __init__(
+ self,
+ user_data_dir: Optional[PathLike] = AUTO,
+ headless: Optional[bool] = False,
+ incognito: Optional[bool] = False,
+ guest: Optional[bool] = False,
+ browser_executable_path: Optional[PathLike] = AUTO,
+ browser_args: Optional[List[str]] = AUTO,
+ sandbox: Optional[bool] = True,
+ lang: Optional[str] = "en-US",
+ host: str = AUTO,
+ port: int = AUTO,
+ expert: bool = AUTO,
+ **kwargs: dict,
+ ):
+ """
+ Creates a config object.
+ Can be called without any arguments to generate a best-practice config,
+ which is recommended.
+ Calling the object, eg: myconfig(), returns the list of arguments which
+ are provided to the browser.
+ Additional args can be added using the :py:obj:`~add_argument method`.
+ Instances of this class are usually not instantiated by end users.
+ :param user_data_dir: the data directory to use
+ :param headless: set to True for headless mode
+ :param browser_executable_path:
+ Specify browser executable, instead of using autodetect.
+ :param browser_args: Forwarded to browser executable.
+ Eg: ["--some-chromeparam=somevalue", "some-other-param=someval"]
+ :param sandbox: disables sandbox
+ :param autodiscover_targets: use autodiscovery of targets
+ :param lang:
+ Language string to use other than the default "en-US,en;q=0.9"
+ :param expert: When set to True, "expert" mode is enabled.
+ This adds: --disable-web-security --disable-site-isolation-trials,
+ as well as some scripts and patching useful for debugging.
+ (For example, ensuring shadow-root is always in "open" mode.)
+ :param kwargs:
+ :type user_data_dir: PathLike
+ :type headless: bool
+ :type browser_executable_path: PathLike
+ :type browser_args: list[str]
+ :type sandbox: bool
+ :type lang: str
+ :type kwargs: dict
+ """
+ if not browser_args:
+ browser_args = []
+ if not user_data_dir:
+ self._user_data_dir = temp_profile_dir()
+ self._custom_data_dir = False
+ else:
+ self.user_data_dir = user_data_dir
+ if not browser_executable_path:
+ browser_executable_path = find_chrome_executable()
+ self._browser_args = browser_args
+ self.browser_executable_path = browser_executable_path
+ self.headless = headless
+ self.incognito = incognito
+ self.guest = guest
+ self.sandbox = sandbox
+ self.host = host
+ self.port = port
+ self.expert = expert
+ self._extensions = []
+ # When using posix-ish operating system and running as root,
+ # you must use no_sandbox=True
+ if is_posix and is_root() and sandbox:
+ logger.info("Detected root usage, auto-disabling sandbox mode.")
+ self.sandbox = False
+ self.autodiscover_targets = True
+ self.lang = lang
+ # Other keyword args will be accessible by attribute
+ self.__dict__.update(kwargs)
+ super().__init__()
+ self._default_browser_args = [
+ "--remote-allow-origins=*",
+ "--no-first-run",
+ "--no-service-autorun",
+ "--no-default-browser-check",
+ "--homepage=about:blank",
+ "--no-pings",
+ "--safebrowsing-disable-download-protection",
+ '--simulate-outdated-no-au="Tue, 31 Dec 2099 23:59:59 GMT"',
+ "--password-store=basic",
+ "--deny-permission-prompts",
+ "--disable-infobars",
+ "--disable-breakpad",
+ "--disable-component-update",
+ "--disable-prompt-on-repost",
+ "--disable-password-generation",
+ "--disable-ipc-flooding-protection",
+ "--disable-search-engine-choice-screen",
+ "--disable-backgrounding-occluded-windows",
+ "--disable-client-side-phishing-detection",
+ "--disable-top-sites",
+ "--disable-renderer-backgrounding",
+ "--disable-background-networking",
+ "--disable-dev-shm-usage",
+ "--disable-features=IsolateOrigins,site-per-process,Translate,"
+ "InsecureDownloadWarnings,DownloadBubble,DownloadBubbleV2,"
+ "OptimizationTargetPrediction,OptimizationGuideModelDownloading,"
+ "SidePanelPinning,UserAgentClientHint,PrivacySandboxSettings4",
+ ]
+
+ @property
+ def browser_args(self):
+ return sorted(self._default_browser_args + self._browser_args)
+
+ @property
+ def user_data_dir(self):
+ return self._user_data_dir
+
+ @user_data_dir.setter
+ def user_data_dir(self, path: PathLike):
+ self._user_data_dir = str(path)
+ self._custom_data_dir = True
+
+ @property
+ def uses_custom_data_dir(self) -> bool:
+ return self._custom_data_dir
+
+ def add_extension(self, extension_path: PathLike):
+ """
+ Adds an extension to load. You can set the extension_path to a
+ folder (containing the manifest), or an extension zip file (.crx)
+ :param extension_path:
+ """
+ path = pathlib.Path(extension_path)
+ if not path.exists():
+ raise FileNotFoundError(
+ "Could not find anything here: %s" % str(path)
+ )
+ if path.is_file():
+ tf = tempfile.mkdtemp(
+ prefix="extension_", suffix=secrets.token_hex(4)
+ )
+ with zipfile.ZipFile(path, "r") as z:
+ z.extractall(tf)
+ self._extensions.append(tf)
+ elif path.is_dir():
+ for item in path.rglob("manifest.*"):
+ path = item.parent
+ self._extensions.append(path)
+
+ def __call__(self):
+ # The host and port will be added when starting the browser.
+ # By the time it starts, the port is probably already taken.
+ args = self._default_browser_args.copy()
+ args += ["--user-data-dir=%s" % self.user_data_dir]
+ args += ["--disable-features=IsolateOrigins,site-per-process"]
+ args += ["--disable-session-crashed-bubble"]
+ if self.expert:
+ args += [
+ "--disable-web-security",
+ "--disable-site-isolation-trials",
+ ]
+ if self._browser_args:
+ args.extend([arg for arg in self._browser_args if arg not in args])
+ if self.headless:
+ args.append("--headless=new")
+ if self.incognito:
+ args.append("--incognito")
+ if self.guest:
+ args.append("--guest")
+ if not self.sandbox:
+ args.append("--no-sandbox")
+ if self.host:
+ args.append("--remote-debugging-host=%s" % self.host)
+ if self.port:
+ args.append("--remote-debugging-port=%s" % self.port)
+ return args
+
+ def add_argument(self, arg: str):
+ if any(
+ x in arg.lower()
+ for x in [
+ "headless",
+ "data-dir",
+ "data_dir",
+ "no-sandbox",
+ "no_sandbox",
+ "lang",
+ ]
+ ):
+ raise ValueError(
+ '"%s" is not allowed. Please use one of the '
+ 'attributes of the Config object to set it.'
+ % arg
+ )
+ self._browser_args.append(arg)
+
+ def __repr__(self):
+ s = f"{self.__class__.__name__}"
+ for k, v in ({**self.__dict__, **self.__class__.__dict__}).items():
+ if k[0] == "_":
+ continue
+ if not v:
+ continue
+ if isinstance(v, property):
+ v = getattr(self, k)
+ if callable(v):
+ continue
+ s += f"\n\t{k} = {v}"
+ return s
+
+
+def is_root():
+ """
+ Helper function to determine if the user is trying to launch chrome
+ under linux as root, which needs some alternative handling.
+ """
+ import ctypes
+ import os
+
+ try:
+ return os.getuid() == 0
+ except AttributeError:
+ return ctypes.windll.shell32.IsUserAnAdmin() != 0
+
+
+def temp_profile_dir():
+ """Generate a temp dir (path)"""
+ path = os.path.normpath(tempfile.mkdtemp(prefix="uc_"))
+ return path
+
+
+def find_chrome_executable(return_all=False):
+ """
+ Finds the chrome, beta, canary, chromium executable
+ and returns the disk path.
+ """
+ candidates = []
+ if is_posix:
+ for item in os.environ.get("PATH").split(os.pathsep):
+ for subitem in (
+ "google-chrome",
+ "chromium",
+ "chromium-browser",
+ "chrome",
+ "google-chrome-stable",
+ ):
+ candidates.append(os.sep.join((item, subitem)))
+ if "darwin" in sys.platform:
+ candidates += [
+ "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+ "/Applications/Chromium.app/Contents/MacOS/Chromium",
+ ]
+ else:
+ for item in map(
+ os.environ.get,
+ (
+ "PROGRAMFILES",
+ "PROGRAMFILES(X86)",
+ "LOCALAPPDATA",
+ "PROGRAMW6432",
+ ),
+ ):
+ if item is not None:
+ for subitem in (
+ "Google/Chrome/Application",
+ "Google/Chrome Beta/Application",
+ "Google/Chrome Canary/Application",
+ ):
+ candidates.append(
+ os.sep.join((item, subitem, "chrome.exe"))
+ )
+ rv = []
+ for candidate in candidates:
+ if os.path.exists(candidate) and os.access(candidate, os.X_OK):
+ logger.debug("%s is a valid candidate... " % candidate)
+ rv.append(candidate)
+ else:
+ logger.debug(
+ "%s is not a valid candidate because it doesn't exist "
+ "or isn't an executable."
+ % candidate
+ )
+ winner = None
+ if return_all and rv:
+ return rv
+ if rv and len(rv) > 1:
+ # Assuming the shortest path wins
+ winner = min(rv, key=lambda x: len(x))
+ elif len(rv) == 1:
+ winner = rv[0]
+ if winner:
+ return os.path.normpath(winner)
+ raise FileNotFoundError(
+ "Could not find a valid chrome browser binary. "
+ "Please make sure Chrome is installed. "
+ "Or use the keyword argument: "
+ "'browser_executable_path=/path/to/your/browser'."
+ )
diff --git a/seleniumbase/undetected/cdp_driver/connection.py b/seleniumbase/undetected/cdp_driver/connection.py
new file mode 100644
index 00000000000..81c519b7bca
--- /dev/null
+++ b/seleniumbase/undetected/cdp_driver/connection.py
@@ -0,0 +1,625 @@
+from __future__ import annotations
+import asyncio
+import collections
+import inspect
+import itertools
+import json
+import logging
+import sys
+import types
+from asyncio import iscoroutine, iscoroutinefunction
+from typing import (
+ Generator,
+ Union,
+ Awaitable,
+ Callable,
+ Any,
+ TypeVar,
+)
+import websockets
+from . import cdp_util as util
+import mycdp as cdp
+import mycdp.network
+import mycdp.page
+import mycdp.storage
+import mycdp.runtime
+import mycdp.target
+import mycdp.util
+
+T = TypeVar("T")
+GLOBAL_DELAY = 0.005
+MAX_SIZE: int = 2**28
+PING_TIMEOUT: int = 1800 # 30 minutes
+TargetType = Union[cdp.target.TargetInfo, cdp.target.TargetID]
+logger = logging.getLogger("uc.connection")
+
+
+class ProtocolException(Exception):
+ def __init__(self, *args, **kwargs):
+ self.message = None
+ self.code = None
+ self.args = args
+ if isinstance(args[0], dict):
+ self.message = args[0].get("message", None) # noqa
+ self.code = args[0].get("code", None)
+ elif hasattr(args[0], "to_json"):
+ def serialize(obj, _d=0):
+ res = "\n"
+ for k, v in obj.items():
+ space = "\t" * _d
+ if isinstance(v, dict):
+ res += f"{space}{k}: {serialize(v, _d + 1)}\n"
+ else:
+ res += f"{space}{k}: {v}\n"
+ return res
+ self.message = serialize(args[0].to_json())
+ else:
+ self.message = "| ".join(str(x) for x in args)
+
+ def __str__(self):
+ return f"{self.message} [code: {self.code}]" if self.code else f"{self.message}" # noqa
+
+
+class SettingClassVarNotAllowedException(PermissionError):
+ pass
+
+
+class Transaction(asyncio.Future):
+ __cdp_obj__: Generator = None
+ method: str = None
+ params: dict = None
+ id: int = None
+
+ def __init__(self, cdp_obj: Generator):
+ """
+ :param cdp_obj:
+ """
+ super().__init__()
+ self.__cdp_obj__ = cdp_obj
+ self.connection = None
+ self.method, *params = next(self.__cdp_obj__).values()
+ if params:
+ params = params.pop()
+ self.params = params
+
+ @property
+ def message(self):
+ return json.dumps(
+ {"method": self.method, "params": self.params, "id": self.id}
+ )
+
+ @property
+ def has_exception(self):
+ try:
+ if self.exception():
+ return True
+ except BaseException:
+ return True
+ return False
+
+ def __call__(self, **response: dict):
+ """
+ Parses the response message and marks the future complete.
+ :param response:
+ """
+ if "error" in response:
+ # Set exception and bail out
+ return self.set_exception(ProtocolException(response["error"]))
+ try:
+ # Try to parse the result according to the PyCDP docs.
+ self.__cdp_obj__.send(response["result"])
+ except StopIteration as e:
+ # Exception value holds the parsed response
+ return self.set_result(e.value)
+ raise ProtocolException(
+ "Could not parse the cdp response:\n%s" % response
+ )
+
+ def __repr__(self):
+ success = False if (self.done() and self.has_exception) else True
+ if self.done():
+ status = "finished"
+ else:
+ status = "pending"
+ fmt = (
+ f"<{self.__class__.__name__}\n\t"
+ f"method: {self.method}\n\t"
+ f"status: {status}\n\t"
+ f"success: {success}>"
+ )
+ return fmt
+
+
+class EventTransaction(Transaction):
+ event = None
+ value = None
+
+ def __init__(self, event_object):
+ try:
+ super().__init__(None)
+ except BaseException:
+ pass
+ self.set_result(event_object)
+ self.event = self.value = self.result()
+
+ def __repr__(self):
+ status = "finished"
+ success = False if self.exception() else True
+ event_object = self.result()
+ fmt = (
+ f"{self.__class__.__name__}\n\t"
+ f"event: {event_object.__class__.__module__}.{event_object.__class__.__name__}\n\t" # noqa
+ f"status: {status}\n\t"
+ f"success: {success}>"
+ )
+ return fmt
+
+
+class CantTouchThis(type):
+ def __setattr__(cls, attr, value):
+ """:meta private:"""
+ if attr == "__annotations__":
+ # Fix autodoc
+ return super().__setattr__(attr, value)
+ raise SettingClassVarNotAllowedException(
+ "\n".join(
+ (
+ "don't set '%s' on the %s class directly, "
+ "as those are shared with other objects.",
+ "use `my_object.%s = %s` instead",
+ )
+ )
+ % (attr, cls.__name__, attr, value)
+ )
+
+
+class Connection(metaclass=CantTouchThis):
+ attached: bool = None
+ websocket: websockets.WebSocketClientProtocol
+ _target: cdp.target.TargetInfo
+
+ def __init__(
+ self,
+ websocket_url=None,
+ target=None,
+ _owner=None,
+ **kwargs,
+ ):
+ super().__init__()
+ self._target = target
+ self.__count__ = itertools.count(0)
+ self._owner = _owner
+ self.websocket_url: str = websocket_url
+ self.websocket = None
+ self.mapper = {}
+ self.handlers = collections.defaultdict(list)
+ self.recv_task = None
+ self.enabled_domains = []
+ self._last_result = []
+ self.listener: Listener = None
+ self.__dict__.update(**kwargs)
+
+ @property
+ def target(self) -> cdp.target.TargetInfo:
+ return self._target
+
+ @target.setter
+ def target(self, target: cdp.target.TargetInfo):
+ if not isinstance(target, cdp.target.TargetInfo):
+ raise TypeError(
+ "target must be set to a '%s' but got '%s"
+ % (cdp.target.TargetInfo.__name__, type(target).__name__)
+ )
+ self._target = target
+
+ @property
+ def closed(self):
+ if not self.websocket:
+ return True
+ return self.websocket.closed
+
+ def add_handler(
+ self,
+ event_type_or_domain: Union[type, types.ModuleType],
+ handler: Union[Callable, Awaitable],
+ ):
+ """
+ Add a handler for given event.
+ If event_type_or_domain is a module instead of a type,
+ it will find all available events and add the handler.
+ If you want to receive event updates (eg. network traffic),
+ you can add handlers for those events.
+ Handlers can be regular callback functions
+ or async coroutine functions (and also just lambdas).
+ For example, if you want to check the network traffic:
+ .. code-block::
+ page.add_handler(
+ cdp.network.RequestWillBeSent, lambda event: print(
+ 'network event => %s' % event.request
+ )
+ )
+ Next time there's network traffic, you'll see lots of console output.
+ :param event_type_or_domain:
+ :param handler:
+ """
+ if isinstance(event_type_or_domain, types.ModuleType):
+ for name, obj in inspect.getmembers_static(event_type_or_domain):
+ if name.isupper():
+ continue
+ if not name[0].isupper():
+ continue
+ if not isinstance(obj, type):
+ continue
+ if inspect.isbuiltin(obj):
+ continue
+ self.handlers[obj].append(handler)
+ return
+ self.handlers[event_type_or_domain].append(handler)
+
+ async def aopen(self, **kw):
+ """
+ Opens the websocket connection. Shouldn't be called manually by users.
+ """
+ if not self.websocket or self.websocket.closed:
+ try:
+ self.websocket = await websockets.connect(
+ self.websocket_url,
+ ping_timeout=PING_TIMEOUT,
+ max_size=MAX_SIZE,
+ )
+ self.listener = Listener(self)
+ except (Exception,) as e:
+ logger.debug("Exception during opening of websocket: %s", e)
+ if self.listener:
+ self.listener.cancel()
+ raise
+ if not self.listener or not self.listener.running:
+ self.listener = Listener(self)
+ logger.debug(
+ "\nβ
Opened websocket connection to %s", self.websocket_url
+ )
+ # When a websocket connection is closed (either by error or on purpose)
+ # and reconnected, the registered event listeners (if any), should be
+ # registered again, so the browser sends those events.
+ await self._register_handlers()
+
+ async def aclose(self):
+ """
+ Closes the websocket connection. Shouldn't be called manually by users.
+ """
+ if self.websocket and not self.websocket.closed:
+ if self.listener and self.listener.running:
+ self.listener.cancel()
+ self.enabled_domains.clear()
+ await self.websocket.close()
+ logger.debug(
+ "\nβ Closed websocket connection to %s", self.websocket_url
+ )
+
+ async def sleep(self, t: Union[int, float] = 0.25):
+ await self.update_target()
+ await asyncio.sleep(t)
+
+ def feed_cdp(self, cdp_obj):
+ """
+ Used in specific cases, mostly during cdp.fetch.RequestPaused events,
+ in which the browser literally blocks.
+ By using feed_cdp, you can issue a response without a blocking "await".
+ Note: This method won't cause a response.
+ Note: This is not an async method, just a regular method!
+ :param cdp_obj:
+ """
+ asyncio.ensure_future(self.send(cdp_obj))
+
+ async def wait(self, t: Union[int, float] = None):
+ """
+ Waits until the event listener reports idle
+ (no new events received in certain timespan).
+ When `t` is provided, ensures waiting for `t` seconds, no matter what.
+ :param t:
+ """
+ await self.update_target()
+ loop = asyncio.get_running_loop()
+ start_time = loop.time()
+ try:
+ if isinstance(t, (int, float)):
+ await asyncio.wait_for(self.listener.idle.wait(), timeout=t)
+ while (loop.time() - start_time) < t:
+ await asyncio.sleep(0.1)
+ else:
+ await self.listener.idle.wait()
+ except asyncio.TimeoutError:
+ if isinstance(t, (int, float)):
+ # Explicit time is given, which is now passed, so leave now.
+ return
+ except AttributeError:
+ # No listener created yet.
+ pass
+
+ def __getattr__(self, item):
+ """:meta private:"""
+ try:
+ return getattr(self.target, item)
+ except AttributeError:
+ raise
+
+ async def __aenter__(self):
+ """:meta private:"""
+ return self
+
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
+ """:meta private:"""
+ await self.aclose()
+ if exc_type and exc_val:
+ raise exc_type(exc_val)
+
+ def __await__(self):
+ """
+ Updates targets and wait for event listener to report idle.
+ Idle is reported when no new events are received for 1 second.
+ """
+ return self.wait().__await__()
+
+ async def update_target(self):
+ target_info: cdp.target.TargetInfo = await self.send(
+ cdp.target.get_target_info(self.target_id), _is_update=True
+ )
+ self.target = target_info
+
+ async def send(
+ self,
+ cdp_obj: Generator[dict[str, Any], dict[str, Any], Any],
+ _is_update=False,
+ ) -> Any:
+ """
+ Send a protocol command.
+ The commands are made using any of the cdp..()'s
+ and is used to send custom cdp commands as well.
+ :param cdp_obj: The generator object created by a cdp method
+ :param _is_update: Internal flag
+ Prevents infinite loop by skipping the registeration of handlers
+ when multiple calls to connection.send() are made.
+ """
+ await self.aopen()
+ if not self.websocket or self.closed:
+ return
+ if self._owner:
+ browser = self._owner
+ if browser.config:
+ if browser.config.expert:
+ await self._prepare_expert()
+ if browser.config.headless:
+ await self._prepare_headless()
+ if not self.listener or not self.listener.running:
+ self.listener = Listener(self)
+ try:
+ tx = Transaction(cdp_obj)
+ tx.connection = self
+ if not self.mapper:
+ self.__count__ = itertools.count(0)
+ tx.id = next(self.__count__)
+ self.mapper.update({tx.id: tx})
+ if not _is_update:
+ await self._register_handlers()
+ await self.websocket.send(tx.message)
+ try:
+ return await tx
+ except ProtocolException as e:
+ e.message += f"\ncommand:{tx.method}\nparams:{tx.params}"
+ raise e
+ except Exception:
+ await self.aclose()
+
+ async def _register_handlers(self):
+ """
+ Ensure that for current (event) handlers, the corresponding
+ domain is enabled in the protocol.
+ """
+ # Save a copy of current enabled domains in a variable.
+ # At the end, this variable will hold the domains that
+ # are not represented by handlers, and can be removed.
+ enabled_domains = self.enabled_domains.copy()
+ for event_type in self.handlers.copy():
+ domain_mod = None
+ if len(self.handlers[event_type]) == 0:
+ self.handlers.pop(event_type)
+ continue
+ if isinstance(event_type, type):
+ domain_mod = util.cdp_get_module(event_type.__module__)
+ if domain_mod in self.enabled_domains:
+ # At this point, the domain is being used by a handler, so
+ # remove that domain from temp variable 'enabled_domains'.
+ if domain_mod in enabled_domains:
+ enabled_domains.remove(domain_mod)
+ continue
+ elif domain_mod not in self.enabled_domains:
+ if domain_mod in (cdp.target, cdp.storage):
+ continue
+ try:
+ # Prevent infinite loops.
+ logger.debug("Registered %s", domain_mod)
+ self.enabled_domains.append(domain_mod)
+ await self.send(domain_mod.enable(), _is_update=True)
+ except BaseException: # Don't error before request is sent
+ logger.debug("", exc_info=True)
+ try:
+ self.enabled_domains.remove(domain_mod)
+ except BaseException:
+ logger.debug("NOT GOOD", exc_info=True)
+ continue
+ finally:
+ continue
+ for ed in enabled_domains:
+ # Items still present at this point are unused and need removal.
+ self.enabled_domains.remove(ed)
+
+ async def _prepare_headless(self):
+ return # (This functionality has moved to a new location!)
+
+ async def _prepare_expert(self):
+ if getattr(self, "_prep_expert_done", None):
+ return
+ if self._owner:
+ part1 = "Element.prototype._attachShadow = "
+ part2 = "Element.prototype.attachShadow"
+ parts = part1 + part2
+ await self._send_oneshot(
+ cdp.page.add_script_to_evaluate_on_new_document(
+ """
+ %s;
+ Element.prototype.attachShadow = function () {
+ return this._attachShadow( { mode: "open" } );
+ };
+ """ % parts
+ )
+ )
+ await self._send_oneshot(cdp.page.enable())
+ setattr(self, "_prep_expert_done", True)
+
+ async def _send_oneshot(self, cdp_obj):
+ tx = Transaction(cdp_obj)
+ tx.connection = self
+ tx.id = -2
+ self.mapper.update({tx.id: tx})
+ await self.websocket.send(tx.message)
+ try:
+ # In try/except since if browser connection sends this,
+ # then it raises an exception.
+ return await tx
+ except ProtocolException:
+ pass
+
+
+class Listener:
+ def __init__(self, connection: Connection):
+ self.connection = connection
+ self.history = collections.deque()
+ self.max_history = 1000
+ self.task: asyncio.Future = None
+ is_interactive = getattr(sys, "ps1", sys.flags.interactive)
+ self._time_before_considered_idle = 0.10 if not is_interactive else 0.75 # noqa
+ self.idle = asyncio.Event()
+ self.run()
+
+ def run(self):
+ self.task = asyncio.create_task(self.listener_loop())
+
+ @property
+ def time_before_considered_idle(self):
+ return self._time_before_considered_idle
+
+ @time_before_considered_idle.setter
+ def time_before_considered_idle(self, seconds: Union[int, float]):
+ self._time_before_considered_idle = seconds
+
+ def cancel(self):
+ if self.task and not self.task.cancelled():
+ self.task.cancel()
+
+ @property
+ def running(self):
+ if not self.task:
+ return False
+ if self.task.done():
+ return False
+ return True
+
+ async def listener_loop(self):
+ while True:
+ try:
+ msg = await asyncio.wait_for(
+ self.connection.websocket.recv(),
+ self.time_before_considered_idle,
+ )
+ except asyncio.TimeoutError:
+ self.idle.set()
+ # Pause for a moment.
+ # await asyncio.sleep(self.time_before_considered_idle / 10)
+ continue
+ except (Exception,) as e:
+ logger.debug(
+ "Connection listener exception "
+ "while reading websocket:\n%s", e
+ )
+ break
+ if not self.running:
+ # If we have been cancelled or otherwise stopped running,
+ # then break this loop.
+ break
+ self.idle.clear() # Not "idle" anymore.
+ message = json.loads(msg)
+ if "id" in message:
+ if message["id"] in self.connection.mapper:
+ tx = self.connection.mapper.pop(message["id"])
+ logger.debug(
+ "Got answer for %s (message_id:%d)", tx, message["id"]
+ )
+ tx(**message)
+ else:
+ if message["id"] == -2:
+ tx = self.connection.mapper.get(-2)
+ if tx:
+ tx(**message)
+ continue
+ else:
+ # Probably an event
+ try:
+ event = cdp.util.parse_json_event(message)
+ event_tx = EventTransaction(event)
+ if not self.connection.mapper:
+ self.connection.__count__ = itertools.count(0)
+ event_tx.id = next(self.connection.__count__)
+ self.connection.mapper[event_tx.id] = event_tx
+ except Exception as e:
+ logger.info(
+ "%s: %s during parsing of json from event : %s"
+ % (type(e).__name__, e.args, message),
+ exc_info=True,
+ )
+ continue
+ except KeyError as e:
+ logger.info("KeyError: %s" % e, exc_info=True)
+ continue
+ try:
+ if type(event) in self.connection.handlers:
+ callbacks = self.connection.handlers[type(event)]
+ else:
+ continue
+ if not len(callbacks):
+ continue
+ for callback in callbacks:
+ try:
+ if (
+ iscoroutinefunction(callback)
+ or iscoroutine(callback)
+ ):
+ try:
+ await callback(event, self.connection)
+ except TypeError:
+ await callback(event)
+ else:
+ try:
+ callback(event, self.connection)
+ except TypeError:
+ callback(event)
+ except Exception as e:
+ logger.warning(
+ "Exception in callback %s for event %s => %s",
+ callback,
+ event.__class__.__name__,
+ e,
+ exc_info=True,
+ )
+ raise
+ except asyncio.CancelledError:
+ break
+ except Exception:
+ raise
+ continue
+
+ def __repr__(self):
+ s_idle = "[idle]" if self.idle.is_set() else "[busy]"
+ s_cache_length = f"[cache size: {len(self.history)}]"
+ s_running = f"[running: {self.running}]"
+ s = f"{self.__class__.__name__} {s_running} {s_idle} {s_cache_length}>"
+ return s
diff --git a/seleniumbase/undetected/cdp_driver/element.py b/seleniumbase/undetected/cdp_driver/element.py
new file mode 100644
index 00000000000..80c2081b2ae
--- /dev/null
+++ b/seleniumbase/undetected/cdp_driver/element.py
@@ -0,0 +1,1150 @@
+from __future__ import annotations
+import asyncio
+import json
+import logging
+import pathlib
+import secrets
+import typing
+from . import cdp_util as util
+from ._contradict import ContraDict
+from .config import PathLike
+import mycdp as cdp
+import mycdp.input_
+import mycdp.dom
+import mycdp.overlay
+import mycdp.page
+import mycdp.runtime
+
+logger = logging.getLogger(__name__)
+if typing.TYPE_CHECKING:
+ from .tab import Tab
+
+
+def create(
+ node: cdp.dom.Node,
+ tab: Tab, tree:
+ typing.Optional[cdp.dom.Node] = None
+):
+ """
+ Factory for Elements.
+ This is used with Tab.query_selector(_all).
+ Since we already have the tree,
+ we don't need to fetch it for every single element.
+ :param node: cdp dom node representation
+ :type node: cdp.dom.Node
+ :param tab: the target object to which this element belongs
+ :type tab: Tab
+ :param tree: [Optional] the full node tree to which belongs,
+ enhances performance. When not provided, you need to
+ call `await elem.update()` before using .children / .parent
+ :type tree:
+ """
+ elem = Element(node, tab, tree)
+ return elem
+
+
+class Element:
+ def __init__(
+ self, node: cdp.dom.Node, tab: Tab, tree: cdp.dom.Node = None
+ ):
+ """
+ Represents an (HTML) DOM Element
+ :param node: cdp dom node representation
+ :type node: cdp.dom.Node
+ :param tab: the target object to which this element belongs
+ :type tab: Tab
+ """
+ if not node:
+ raise Exception("Node cannot be None!")
+ self._tab = tab
+ # if node.node_name == 'IFRAME':
+ # self._node = node.content_document
+ self._node = node
+ self._tree = tree
+ self._parent = None
+ self._remote_object = None
+ self._attrs = ContraDict(silent=True)
+ self._make_attrs()
+
+ @property
+ def tag(self):
+ if self.node_name:
+ return self.node_name.lower()
+
+ @property
+ def tag_name(self):
+ return self.tag
+
+ @property
+ def node_id(self):
+ return self.node.node_id
+
+ @property
+ def backend_node_id(self):
+ return self.node.backend_node_id
+
+ @property
+ def node_type(self):
+ return self.node.node_type
+
+ @property
+ def node_name(self):
+ return self.node.node_name
+
+ @property
+ def local_name(self):
+ return self.node.local_name
+
+ @property
+ def node_value(self):
+ return self.node.node_value
+
+ @property
+ def parent_id(self):
+ return self.node.parent_id
+
+ @property
+ def child_node_count(self):
+ return self.node.child_node_count
+
+ @property
+ def attributes(self):
+ return self.node.attributes
+
+ @property
+ def document_url(self):
+ return self.node.document_url
+
+ @property
+ def base_url(self):
+ return self.node.base_url
+
+ @property
+ def public_id(self):
+ return self.node.public_id
+
+ @property
+ def system_id(self):
+ return self.node.system_id
+
+ @property
+ def internal_subset(self):
+ return self.node.internal_subset
+
+ @property
+ def xml_version(self):
+ return self.node.xml_version
+
+ @property
+ def value(self):
+ return self.node.value
+
+ @property
+ def pseudo_type(self):
+ return self.node.pseudo_type
+
+ @property
+ def pseudo_identifier(self):
+ return self.node.pseudo_identifier
+
+ @property
+ def shadow_root_type(self):
+ return self.node.shadow_root_type
+
+ @property
+ def frame_id(self):
+ return self.node.frame_id
+
+ @property
+ def content_document(self):
+ return self.node.content_document
+
+ @property
+ def shadow_roots(self):
+ return self.node.shadow_roots
+
+ @property
+ def template_content(self):
+ return self.node.template_content
+
+ @property
+ def pseudo_elements(self):
+ return self.node.pseudo_elements
+
+ @property
+ def imported_document(self):
+ return self.node.imported_document
+
+ @property
+ def distributed_nodes(self):
+ return self.node.distributed_nodes
+
+ @property
+ def is_svg(self):
+ return self.node.is_svg
+
+ @property
+ def compatibility_mode(self):
+ return self.node.compatibility_mode
+
+ @property
+ def assigned_slot(self):
+ return self.node.assigned_slot
+
+ @property
+ def tab(self):
+ return self._tab
+
+ def __getattr__(self, item):
+ # If attribute is not found on the element object,
+ # check if it is present in the element attributes
+ # (Eg. href=, src=, alt=).
+ # Returns None when attribute is not found,
+ # instead of raising AttributeError.
+ x = getattr(self.attrs, item, None)
+ if x:
+ return x
+
+ def __setattr__(self, key, value):
+ if key[0] != "_":
+ if key[1:] not in vars(self).keys():
+ self.attrs.__setattr__(key, value)
+ return
+ super().__setattr__(key, value)
+
+ def __setitem__(self, key, value):
+ if key[0] != "_":
+ if key[1:] not in vars(self).keys():
+ self.attrs[key] = value
+
+ def __getitem__(self, item):
+ return self.attrs.get(item, None)
+
+ async def save_to_dom_async(self):
+ """Saves element to DOM."""
+ self._remote_object = await self._tab.send(
+ cdp.dom.resolve_node(backend_node_id=self.backend_node_id)
+ )
+ await self._tab.send(
+ cdp.dom.set_outer_html(self.node_id, outer_html=str(self))
+ )
+ await self.update()
+
+ async def remove_from_dom_async(self):
+ """Removes element from DOM."""
+ await self.update() # Ensure we have latest node_id
+ node = util.filter_recurse(
+ self._tree,
+ lambda node: node.backend_node_id == self.backend_node_id
+ )
+ if node:
+ await self.tab.send(cdp.dom.remove_node(node.node_id))
+ # self._tree = util.remove_from_tree(self.tree, self.node)
+
+ async def update(self, _node=None):
+ """
+ Updates element to retrieve more properties.
+ For example this enables:
+ :py:obj:`~children` and :py:obj:`~parent` attributes.
+ Also resolves js object,
+ which is a stored object in :py:obj:`~remote_object`.
+ Usually you will get element nodes by the usage of:
+ :py:meth:`Tab.query_selector_all()`
+ :py:meth:`Tab.find_elements_by_text()`
+ Those elements are already updated and you can browse
+ through children directly.
+ """
+ if _node:
+ doc = _node
+ # self._node = _node
+ # self._children.clear()
+ self._parent = None
+ else:
+ doc = await self._tab.send(cdp.dom.get_document(-1, True))
+ self._parent = None
+ # if self.node_name != "IFRAME":
+ updated_node = util.filter_recurse(
+ doc, lambda n: n.backend_node_id == self._node.backend_node_id
+ )
+ if updated_node:
+ logger.debug("Node changed, and has now been updated.")
+ self._node = updated_node
+ self._tree = doc
+ self._remote_object = await self._tab.send(
+ cdp.dom.resolve_node(backend_node_id=self._node.backend_node_id)
+ )
+ # self.attrs.clear()
+ self._make_attrs()
+ if self.node_name != "IFRAME":
+ parent_node = util.filter_recurse(
+ doc, lambda n: n.node_id == self.node.parent_id
+ )
+ if not parent_node:
+ # Could happen if node is for example
+ return self
+ self._parent = create(parent_node, tab=self._tab, tree=self._tree)
+ return self
+
+ @property
+ def node(self):
+ return self._node
+
+ @property
+ def tree(self) -> cdp.dom.Node:
+ return self._tree
+
+ @tree.setter
+ def tree(self, tree: cdp.dom.Node):
+ self._tree = tree
+
+ @property
+ def attrs(self):
+ """
+ Attributes are stored here.
+ You can also set them directly on the element object.
+ """
+ return self._attrs
+
+ @property
+ def parent(self) -> typing.Union[Element, None]:
+ """Get the parent element (node) of current element (node)."""
+ if not self.tree:
+ raise RuntimeError(
+ "Could not get parent since the element has no tree set."
+ )
+ parent_node = util.filter_recurse(
+ self.tree, lambda n: n.node_id == self.parent_id
+ )
+ if not parent_node:
+ return None
+ parent_element = create(parent_node, tab=self._tab, tree=self.tree)
+ return parent_element
+
+ @property
+ def children(self) -> typing.Union[typing.List[Element], str]:
+ """
+ Returns the element's children.
+ Those children also have a children property
+ so that you can browse through the entire tree as well.
+ """
+ _children = []
+ if self._node.node_name == "IFRAME":
+ # iframes are not the same as other nodes.
+ # The children of iframes are found under
+ # the .content_document property,
+ # which is more useful than the node itself.
+ frame = self._node.content_document
+ if not frame.child_node_count:
+ return []
+ for child in frame.children:
+ child_elem = create(child, self._tab, frame)
+ if child_elem:
+ _children.append(child_elem)
+ # self._node = frame
+ return _children
+ elif not self.node.child_node_count:
+ return []
+ if self.node.children:
+ for child in self.node.children:
+ child_elem = create(child, self._tab, self.tree)
+ if child_elem:
+ _children.append(child_elem)
+ return _children
+
+ @property
+ def remote_object(self) -> cdp.runtime.RemoteObject:
+ return self._remote_object
+
+ @property
+ def object_id(self) -> cdp.runtime.RemoteObjectId:
+ try:
+ return self.remote_object.object_id
+ except AttributeError:
+ pass
+
+ async def click_async(self):
+ """Click the element."""
+ self._remote_object = await self._tab.send(
+ cdp.dom.resolve_node(
+ backend_node_id=self.backend_node_id
+ )
+ )
+ arguments = [cdp.runtime.CallArgument(
+ object_id=self._remote_object.object_id
+ )]
+ await self.flash_async(0.25)
+ await self._tab.send(
+ cdp.runtime.call_function_on(
+ "(el) => el.click()",
+ object_id=self._remote_object.object_id,
+ arguments=arguments,
+ await_promise=True,
+ user_gesture=True,
+ return_by_value=True,
+ )
+ )
+
+ async def get_js_attributes_async(self):
+ return ContraDict(
+ json.loads(
+ await self.apply(
+ """
+ function (e) {
+ let o = {}
+ for(let k in e){
+ o[k] = e[k]
+ }
+ return JSON.stringify(o)
+ }
+ """
+ )
+ )
+ )
+
+ def __await__(self):
+ return self.update().__await__()
+
+ def __call__(self, js_method):
+ return self.apply(f"(e) => e['{js_method}']()")
+
+ async def apply(self, js_function, return_by_value=True):
+ """
+ Apply javascript to this element.
+ The given js_function string should accept the js element as parameter,
+ and can be a arrow function, or function declaration.
+ Eg:
+ - '(elem) => {
+ elem.value = "blabla"; consolelog(elem);
+ alert(JSON.stringify(elem);
+ } '
+ - 'elem => elem.play()'
+ - function myFunction(elem) { alert(elem) }
+ :param js_function: JS function definition which received this element.
+ :param return_by_value:
+ """
+ self._remote_object = await self._tab.send(
+ cdp.dom.resolve_node(backend_node_id=self.backend_node_id)
+ )
+ result: typing.Tuple[cdp.runtime.RemoteObject, typing.Any] = (
+ await self._tab.send(
+ cdp.runtime.call_function_on(
+ js_function,
+ object_id=self._remote_object.object_id,
+ arguments=[
+ cdp.runtime.CallArgument(
+ object_id=self._remote_object.object_id
+ )
+ ],
+ return_by_value=True,
+ user_gesture=True,
+ )
+ )
+ )
+ if result and result[0]:
+ if return_by_value:
+ return result[0].value
+ return result[0]
+ elif result[1]:
+ return result[1]
+
+ async def get_position_async(self, abs=False) -> Position:
+ if not self.parent or not self.object_id:
+ self._remote_object = await self._tab.send(
+ cdp.dom.resolve_node(backend_node_id=self.backend_node_id)
+ )
+ # await self.update()
+ try:
+ quads = await self.tab.send(
+ cdp.dom.get_content_quads(
+ object_id=self.remote_object.object_id
+ )
+ )
+ if not quads:
+ raise Exception("Could not find position for %s " % self)
+ pos = Position(quads[0])
+ if abs:
+ scroll_y = (await self.tab.evaluate("window.scrollY")).value
+ scroll_x = (await self.tab.evaluate("window.scrollX")).value
+ abs_x = pos.left + scroll_x + (pos.width / 2)
+ abs_y = pos.top + scroll_y + (pos.height / 2)
+ pos.abs_x = abs_x
+ pos.abs_y = abs_y
+ return pos
+ except IndexError:
+ logger.debug(
+ "No content quads for %s. "
+ "Mostly caused by element which is not 'in plain sight'."
+ % self
+ )
+
+ async def mouse_click_async(
+ self,
+ button: str = "left",
+ buttons: typing.Optional[int] = 1,
+ modifiers: typing.Optional[int] = 0,
+ hold: bool = False,
+ _until_event: typing.Optional[type] = None,
+ ):
+ """
+ Native click (on element).
+ Note: This likely does not work at the moment. Use click() instead.
+ :param button: str (default = "left")
+ :param buttons: which button (default 1 = left)
+ :param modifiers: *(Optional)*
+ Bit field representing pressed modifier keys.
+ Alt=1, Ctrl=2, Meta/Command=4, Shift=8 (default: 0).
+ :param _until_event: Internal. Event to wait for before returning.
+ """
+ try:
+ center = (await self.get_position_async()).center
+ except AttributeError:
+ return
+ if not center:
+ logger.warning("Could not calculate box model for %s", self)
+ return
+ logger.debug("Clicking on location: %.2f, %.2f" % center)
+ await asyncio.gather(
+ self._tab.send(
+ cdp.input_.dispatch_mouse_event(
+ "mousePressed",
+ x=center[0],
+ y=center[1],
+ modifiers=modifiers,
+ button=cdp.input_.MouseButton(button),
+ buttons=buttons,
+ click_count=1,
+ )
+ ),
+ self._tab.send(
+ cdp.input_.dispatch_mouse_event(
+ "mouseReleased",
+ x=center[0],
+ y=center[1],
+ modifiers=modifiers,
+ button=cdp.input_.MouseButton(button),
+ buttons=buttons,
+ click_count=1,
+ )
+ ),
+ )
+ try:
+ await self.flash_async()
+ except BaseException:
+ pass
+
+ async def mouse_move_async(self):
+ """
+ Moves the mouse to the element position.
+ When an element has an hover/mouseover effect, this triggers it.
+ """
+ try:
+ center = (await self.get_position_async()).center
+ except AttributeError:
+ logger.debug("Did not find location for %s", self)
+ return
+ logger.debug(
+ "Mouse move to location %.2f, %.2f where %s is located",
+ *center,
+ self,
+ )
+ await self._tab.send(
+ cdp.input_.dispatch_mouse_event(
+ "mouseMoved", x=center[0], y=center[1]
+ )
+ )
+ await self._tab.sleep(0.05)
+ await self._tab.send(
+ cdp.input_.dispatch_mouse_event(
+ "mouseReleased", x=center[0], y=center[1]
+ )
+ )
+
+ async def mouse_drag_async(
+ self,
+ destination: typing.Union[Element, typing.Tuple[int, int]],
+ relative: bool = False,
+ steps: int = 1,
+ ):
+ """
+ Drags an element to another element or target coordinates.
+ Dragging of elements should be supported by the site.
+ :param destination: Another element where to drag to,
+ or a tuple (x,y) of ints representing coordinates.
+ :type destination: Element or coordinate as x,y tuple
+ :param relative: when True, treats coordinate as relative.
+ For example (-100, 200) will move left 100px and down 200px.
+ :type relative:
+ :param steps: Move in points.
+ This could make it look more "natural" (default 1),
+ but also a lot slower. (For very smooth actions, use 50-100)
+ :type steps: int
+ """
+ try:
+ start_point = (await self.get_position_async()).center
+ except AttributeError:
+ return
+ if not start_point:
+ logger.warning("Could not calculate box model for %s", self)
+ return
+ end_point = None
+ if isinstance(destination, Element):
+ try:
+ end_point = (await destination.get_position_async()).center
+ except AttributeError:
+ return
+ if not end_point:
+ logger.warning(
+ "Could not calculate box model for %s", destination
+ )
+ return
+ elif isinstance(destination, (tuple, list)):
+ if relative:
+ end_point = (
+ start_point[0] + destination[0],
+ start_point[1] + destination[1],
+ )
+ else:
+ end_point = destination
+ await self._tab.send(
+ cdp.input_.dispatch_mouse_event(
+ "mousePressed",
+ x=start_point[0],
+ y=start_point[1],
+ button=cdp.input_.MouseButton("left"),
+ )
+ )
+ steps = 1 if (not steps or steps < 1) else steps
+ if steps == 1:
+ await self._tab.send(
+ cdp.input_.dispatch_mouse_event(
+ "mouseMoved",
+ x=end_point[0],
+ y=end_point[1],
+ )
+ )
+ elif steps > 1:
+ step_size_x = (end_point[0] - start_point[0]) / steps
+ step_size_y = (end_point[1] - start_point[1]) / steps
+ pathway = [
+ (
+ start_point[0] + step_size_x * i,
+ start_point[1] + step_size_y * i,
+ )
+ for i in range(steps + 1)
+ ]
+ for point in pathway:
+ await self._tab.send(
+ cdp.input_.dispatch_mouse_event(
+ "mouseMoved",
+ x=point[0],
+ y=point[1],
+ )
+ )
+ await asyncio.sleep(0)
+ await self._tab.send(
+ cdp.input_.dispatch_mouse_event(
+ type_="mouseReleased",
+ x=end_point[0],
+ y=end_point[1],
+ button=cdp.input_.MouseButton("left"),
+ )
+ )
+
+ async def scroll_into_view_async(self):
+ """Scrolls element into view."""
+ try:
+ await self.tab.send(
+ cdp.dom.scroll_into_view_if_needed(
+ backend_node_id=self.backend_node_id
+ )
+ )
+ except Exception as e:
+ logger.debug("Could not scroll into view: %s", e)
+ return
+ # await self.apply("""(el) => el.scrollIntoView(false)""")
+
+ async def clear_input_async(self, _until_event: type = None):
+ """Clears an input field."""
+ try:
+ await self.apply('function (element) { element.value = "" } ')
+ except Exception as e:
+ logger.debug("Could not clear element field: %s", e)
+ return
+
+ async def send_keys_async(self, text: str):
+ """
+ Send text to an input field, or any other html element.
+ Hint: If you ever get stuck where using py:meth:`~click`
+ does not work, sending the keystroke \\n or \\r\\n
+ or a spacebar works wonders!
+ :param text: text to send
+ :return: None
+ """
+ await self.apply("(elem) => elem.focus()")
+ [
+ await self._tab.send(
+ cdp.input_.dispatch_key_event("char", text=char)
+ )
+ for char in list(text)
+ ]
+
+ async def send_file_async(self, *file_paths: PathLike):
+ """
+ Some form input require a file (upload).
+ A full path needs to be provided.
+ This method sends 1 or more file(s) to the input field.
+ Make sure the field accepts multiple files in order to send more files.
+ (Otherwise the browser might crash.)
+ Example:
+ `await fileinputElement.send_file('c:/tmp/img.png', 'c:/dir/lol.gif')`
+ """
+ file_paths = [str(p) for p in file_paths]
+ await self._tab.send(
+ cdp.dom.set_file_input_files(
+ files=[*file_paths],
+ backend_node_id=self.backend_node_id,
+ object_id=self.object_id,
+ )
+ )
+
+ async def focus_async(self):
+ """Focus the current element. Often useful in form (select) fields."""
+ return await self.apply("(element) => element.focus()")
+
+ async def select_option_async(self):
+ """
+ For form (select) fields. When you have queried the options
+ you can call this method on the option object.
+ Calling :func:`option.select_option()` uses option as selected value.
+ (Does not work in all cases.)
+ """
+ if self.node_name == "OPTION":
+ await self.apply(
+ """
+ (o) => {
+ o.selected = true;
+ o.dispatchEvent(new Event(
+ 'change', {view: window,bubbles: true})
+ )
+ }
+ """
+ )
+
+ async def set_value_async(self, value):
+ await self._tab.send(
+ cdp.dom.set_node_value(node_id=self.node_id, value=value)
+ )
+
+ async def set_text_async(self, value):
+ if not self.node_type == 3:
+ if self.child_node_count == 1:
+ child_node = self.children[0]
+ await child_node.set_text_async(value)
+ await self.update()
+ return
+ else:
+ raise RuntimeError("Could only set value of text nodes.")
+ await self.update()
+ await self._tab.send(
+ cdp.dom.set_node_value(node_id=self.node_id, value=value)
+ )
+
+ async def get_html_async(self):
+ return await self._tab.send(
+ cdp.dom.get_outer_html(backend_node_id=self.backend_node_id)
+ )
+
+ @property
+ def text_fragment(self) -> str:
+ """Gets the text content of this specific element node."""
+ text_node = util.filter_recurse(self.node, lambda n: n.node_type == 3)
+ if text_node:
+ return text_node.node_value.strip()
+ return ""
+
+ @property
+ def text(self):
+ """
+ Gets the text contents of this element and child nodes, concatenated.
+ Note: This includes text in the form of script content, (text nodes).
+ """
+ text_nodes = util.filter_recurse_all(
+ self.node, lambda n: n.node_type == 3
+ )
+ return " ".join([n.node_value for n in text_nodes]).strip()
+
+ @property
+ def text_all(self):
+ """Same as text(). Kept for backwards compatibility."""
+ text_nodes = util.filter_recurse_all(
+ self.node, lambda n: n.node_type == 3
+ )
+ return " ".join([n.node_value for n in text_nodes]).strip()
+
+ async def query_selector_all_async(self, selector: str):
+ """Like JS querySelectorAll()"""
+ await self.update()
+ return await self.tab.query_selector_all(selector, _node=self)
+
+ async def query_selector_async(self, selector: str):
+ """Like JS querySelector()"""
+ await self.update()
+ return await self.tab.query_selector(selector, self)
+
+ async def save_screenshot_async(
+ self,
+ filename: typing.Optional[PathLike] = "auto",
+ format: typing.Optional[str] = "png",
+ scale: typing.Optional[typing.Union[int, float]] = 1,
+ ):
+ """
+ Saves a screenshot of this element (only).
+ This is not the same as :py:obj:`Tab.save_screenshot`,
+ which saves a "regular" screenshot.
+ When the element is hidden, or has no size,
+ or is otherwise not capturable, a RuntimeError is raised.
+ :param filename: uses this as the save path
+ :type filename: PathLike
+ :param format: jpeg or png (defaults to png)
+ :type format: str
+ :param scale: the scale of the screenshot,
+ eg: 1 = size as is, 2 = double, 0.5 is half.
+ :return: the path/filename of saved screenshot
+ :rtype: str
+ """
+ import urllib.parse
+ import datetime
+ import base64
+
+ pos = await self.get_position_async()
+ if not pos:
+ raise RuntimeError(
+ "Could not determine position of element. "
+ "Probably because it's not in view, or hidden."
+ )
+ viewport = pos.to_viewport(scale)
+ path = None
+ await self.tab.sleep()
+ if not filename or filename == "auto":
+ parsed = urllib.parse.urlparse(self.tab.target.url)
+ parts = parsed.path.split("/")
+ last_part = parts[-1]
+ last_part = last_part.rsplit("?", 1)[0]
+ dt_str = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+ candidate = f"{parsed.hostname}__{last_part}_{dt_str}"
+ ext = ""
+ if format.lower() in ["jpg", "jpeg"]:
+ ext = ".jpg"
+ format = "jpeg"
+ elif format.lower() in ["png"]:
+ ext = ".png"
+ format = "png"
+ path = pathlib.Path(candidate + ext)
+ else:
+ if filename.lower().endswith(".png"):
+ format = "png"
+ elif (
+ filename.lower().endswith(".jpg")
+ or filename.lower().endswith(".jpeg")
+ ):
+ format = "jpeg"
+ path = pathlib.Path(filename)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ data = await self._tab.send(
+ cdp.page.capture_screenshot(
+ format, clip=viewport, capture_beyond_viewport=True
+ )
+ )
+ if not data:
+ from .connection import ProtocolException
+
+ raise ProtocolException(
+ "Could not take screenshot. "
+ "Most possible cause is the page has not finished loading yet."
+ )
+ data_bytes = base64.b64decode(data)
+ if not path:
+ raise RuntimeError("Invalid filename or path: '%s'" % filename)
+ path.write_bytes(data_bytes)
+ return str(path)
+
+ async def flash_async(self, duration: typing.Union[float, int] = 0.5):
+ """
+ Displays for a short time a red dot on the element.
+ (Only if the element itself is visible)
+ :param coords: x,y
+ :param duration: seconds (default 0.5)
+ """
+ from .connection import ProtocolException
+
+ if not self.remote_object:
+ try:
+ self._remote_object = await self.tab.send(
+ cdp.dom.resolve_node(backend_node_id=self.backend_node_id)
+ )
+ except ProtocolException:
+ return
+ try:
+ pos = await self.get_position_async()
+ except (Exception,):
+ logger.debug("flash() : Could not determine position.")
+ return
+ style = (
+ "position:absolute;z-index:99999999;padding:0;margin:0;"
+ "left:{:.1f}px; top: {:.1f}px; opacity:0.7;"
+ "width:8px;height:8px;border-radius:50%;background:#EE4488;"
+ "animation:show-pointer-ani {:.2f}s ease 1;"
+ ).format(
+ pos.center[0] - 4, # -4 to account for drawn circle itself (w,h)
+ pos.center[1] - 4,
+ duration,
+ )
+ script = (
+ """
+ (targetElement) => {{
+ var css = document.styleSheets[0];
+ for( let css of [...document.styleSheets]) {{
+ try {{
+ css.insertRule(`
+ @keyframes show-pointer-ani {{
+ 0% {{ opacity: 1; transform: scale(2, 2);}}
+ 25% {{ transform: scale(5,5) }}
+ 50% {{ transform: scale(3, 3);}}
+ 75%: {{ transform: scale(2,2) }}
+ 100% {{ transform: scale(1, 1); opacity: 0;}}
+ }}`,css.cssRules.length);
+ break;
+ }} catch (e) {{
+ console.log(e)
+ }}
+ }};
+ var _d = document.createElement('div');
+ _d.style = `{0:s}`;
+ _d.id = `{1:s}`;
+ document.body.insertAdjacentElement('afterBegin', _d);
+ setTimeout(
+ () => document.getElementById('{1:s}').remove(), {2:d}
+ );
+ }}
+ """.format(
+ style,
+ secrets.token_hex(8),
+ int(duration * 1000),
+ )
+ .replace(" ", "")
+ .replace("\n", "")
+ )
+ arguments = [cdp.runtime.CallArgument(
+ object_id=self._remote_object.object_id
+ )]
+ await self._tab.send(
+ cdp.runtime.call_function_on(
+ script,
+ object_id=self._remote_object.object_id,
+ arguments=arguments,
+ await_promise=True,
+ user_gesture=True,
+ )
+ )
+
+ async def highlight_overlay_async(self):
+ """
+ Highlights the element devtools-style.
+ To remove the highlight, call the method again.
+ """
+ if getattr(self, "_is_highlighted", False):
+ del self._is_highlighted
+ await self.tab.send(cdp.overlay.hide_highlight())
+ await self.tab.send(cdp.dom.disable())
+ await self.tab.send(cdp.overlay.disable())
+ return
+ await self.tab.send(cdp.dom.enable())
+ await self.tab.send(cdp.overlay.enable())
+ conf = cdp.overlay.HighlightConfig(
+ show_info=True, show_extension_lines=True, show_styles=True
+ )
+ await self.tab.send(
+ cdp.overlay.highlight_node(
+ highlight_config=conf, backend_node_id=self.backend_node_id
+ )
+ )
+ setattr(self, "_is_highlighted", 1)
+
+ async def record_video_async(
+ self,
+ filename: typing.Optional[str] = None,
+ folder: typing.Optional[str] = None,
+ duration: typing.Optional[typing.Union[int, float]] = None,
+ ):
+ """
+ Experimental option.
+ :param filename: the desired filename
+ :param folder: the download folder path
+ :param duration: record for this many seconds and then download
+ On html5 video nodes,
+ you can call this method to start recording of the video.
+ When any of the follow happens, the video recorded will be downloaded:
+ - video ends
+ - calling videoelement('pause')
+ - video stops
+ """
+ if self.node_name != "VIDEO":
+ raise RuntimeError(
+ "record_video() can only be called on html5 video elements"
+ )
+ if not folder:
+ directory_path = pathlib.Path.cwd() / "downloads"
+ else:
+ directory_path = pathlib.Path(folder)
+ directory_path.mkdir(exist_ok=True)
+ await self._tab.send(
+ cdp.browser.set_download_behavior(
+ "allow", download_path=str(directory_path)
+ )
+ )
+ await self("pause")
+ dtm = 'document.title + ".mp4"'
+ await self.apply(
+ """
+ function extractVid(vid) {{
+ var duration = {duration:.1f};
+ var stream = vid.captureStream();
+ var mr = new MediaRecorder(
+ stream, {{audio:true, video:true}}
+ )
+ mr.ondataavailable = function(e) {{
+ vid['_recording'] = false
+ var blob = e.data;
+ f = new File(
+ [blob], {{name: {filename}, type:'octet/stream'}}
+ );
+ var objectUrl = URL.createObjectURL(f);
+ var link = document.createElement('a');
+ link.setAttribute('href', objectUrl)
+ link.setAttribute('download', {filename})
+ link.style.display = 'none'
+ document.body.appendChild(link)
+ link.click()
+ document.body.removeChild(link)
+ }}
+ mr.start()
+ vid.addEventListener('ended' , (e) => mr.stop())
+ vid.addEventListener('pause' , (e) => mr.stop())
+ vid.addEventListener('abort', (e) => mr.stop())
+ if ( duration ) {{
+ setTimeout(
+ () => {{ vid.pause(); vid.play() }}, duration
+ );
+ }}
+ vid['_recording'] = true
+ ;}}
+ """.format(
+ filename=f'"{filename}"' if filename else dtm,
+ duration=int(duration * 1000) if duration else 0,
+ )
+ )
+ await self("play")
+ await self._tab
+
+ async def is_recording_async(self):
+ return await self.apply('(vid) => vid["_recording"]')
+
+ def _make_attrs(self):
+ sav = None
+ if self.node.attributes:
+ for i, a in enumerate(self.node.attributes):
+ if i == 0 or i % 2 == 0:
+ if a == "class":
+ a = "class_"
+ sav = a
+ else:
+ if sav:
+ self.attrs[sav] = a
+
+ def __eq__(self, other: Element) -> bool:
+ # if other.__dict__.values() == self.__dict__.values():
+ # return True
+ if other.backend_node_id and self.backend_node_id:
+ return other.backend_node_id == self.backend_node_id
+ return False
+
+ def __repr__(self):
+ tag_name = self.node.node_name.lower()
+ content = ""
+ # Collect all text from this leaf.
+ if self.child_node_count:
+ if self.child_node_count == 1:
+ if self.children:
+ content += str(self.children[0])
+ elif self.child_node_count > 1:
+ if self.children:
+ for child in self.children:
+ content += str(child)
+ if self.node.node_type == 3: # Could be a text node
+ content += self.node_value
+ # Return text only. (No tag names)
+ # This makes it look most natural.
+ return content
+ attrs = " ".join(
+ [
+ f'{k if k != "class_" else "class"}="{v}"'
+ for k, v in self.attrs.items()
+ ]
+ )
+ s = f"<{tag_name} {attrs}>{content}"
+ return s
+
+
+class Position(cdp.dom.Quad):
+ """Helper class for element-positioning."""
+
+ def __init__(self, points):
+ super().__init__(points)
+ (
+ self.left,
+ self.top,
+ self.right,
+ self.top,
+ self.right,
+ self.bottom,
+ self.left,
+ self.bottom,
+ ) = points
+ self.abs_x: float = 0
+ self.abs_y: float = 0
+ self.x = self.left
+ self.y = self.top
+ self.height, self.width = (
+ self.bottom - self.top, self.right - self.left
+ )
+ self.center = (
+ self.left + (self.width / 2),
+ self.top + (self.height / 2),
+ )
+
+ def to_viewport(self, scale=1):
+ return cdp.page.Viewport(
+ x=self.x,
+ y=self.y,
+ width=self.width,
+ height=self.height,
+ scale=scale,
+ )
+
+ def __repr__(self):
+ return (
+ f"""
+ """
+ )
+
+
+async def resolve_node(tab: Tab, node_id: cdp.dom.NodeId):
+ remote_obj: cdp.runtime.RemoteObject = await tab.send(
+ cdp.dom.resolve_node(node_id=node_id)
+ )
+ node_id: cdp.dom.NodeId = await tab.send(cdp.dom.request_node(
+ remote_obj.object_id
+ ))
+ node: cdp.dom.Node = await tab.send(cdp.dom.describe_node(node_id))
+ return node
diff --git a/seleniumbase/undetected/cdp_driver/tab.py b/seleniumbase/undetected/cdp_driver/tab.py
new file mode 100644
index 00000000000..6cdf9b8334b
--- /dev/null
+++ b/seleniumbase/undetected/cdp_driver/tab.py
@@ -0,0 +1,1319 @@
+from __future__ import annotations
+import asyncio
+import logging
+import pathlib
+import warnings
+from typing import Dict, List, Union, Optional, Tuple
+from . import browser as cdp_browser
+from . import element
+from . import cdp_util as util
+from .config import PathLike
+from .connection import Connection, ProtocolException
+import mycdp as cdp
+
+logger = logging.getLogger(__name__)
+
+
+class Tab(Connection):
+ """
+ :ref:`tab` is the controlling mechanism/connection to a 'target',
+ for most of us 'target' can be read as 'tab'. However it could also
+ be an iframe, serviceworker or background script for example,
+ although there isn't much to control for those.
+ If you open a new window by using
+ :py:meth:`browser.get(..., new_window=True)`
+ Your url will open a new window. This window is a 'tab'.
+ When you browse to another page, the tab will be the same (browser view).
+ It's important to keep some reference to tab objects, in case you're
+ done interacting with elements and want to operate on the page level again.
+
+ Custom CDP commands
+ ---------------------------
+ Tab object provide many useful and often-used methods. It is also possible
+ to utilize the included cdp classes to to something totally custom.
+
+ The cdp package is a set of so-called "domains" with each having methods,
+ events and types.
+ To send a cdp method, for example :py:obj:`cdp.page.navigate`,
+ you'll have to check whether the method accepts any parameters
+ and whether they are required or not.
+
+ You can use:
+
+ ```python
+ await tab.send(cdp.page.navigate(url='https://Your-URL-Here'))
+ ```
+
+ So tab.send() accepts a generator object,
+ which is created by calling a cdp method.
+ This way you can build very detailed and customized commands.
+ (Note: Finding correct command combos can be a time-consuming task.
+ A whole bunch of useful methods have been added,
+ preferably having the same apis or lookalikes, as in selenium.)
+
+ Some useful, often needed and simply required methods
+ ===================================================================
+
+ :py:meth:`~find` | find(text)
+ ----------------------------------------
+ Finds and returns a single element by text match.
+ By default, returns the first element found.
+ Much more powerful is the best_match flag,
+ although also much more expensive.
+ When no match is found, it will retry for seconds (default: 10),
+ so this is also suitable to use as wait condition.
+
+ :py:meth:`~find` | find(text, best_match=True) or find(text, True)
+ -----------------------------------------------------------------------
+ Much more powerful (and expensive) than the above is
+ the use of the `find(text, best_match=True)` flag.
+ It will still return 1 element, but when multiple matches are found,
+ it picks the one having the most similar text length.
+ How would that help?
+ For example, you search for "login",
+ you'd probably want the "login" button element,
+ and not thousands of scripts/meta/headings,
+ which happens to contain a string of "login".
+
+ When no match is found, it will retry for seconds (default: 10),
+ so this is also suitable to use as wait condition.
+
+ :py:meth:`~select` | select(selector)
+ ----------------------------------------
+ Finds and returns a single element by css selector match.
+ When no match is found, it will retry for seconds (default: 10),
+ so this is also suitable to use as wait condition.
+
+ :py:meth:`~select_all` | select_all(selector)
+ ------------------------------------------------
+ Finds and returns all elements by css selector match.
+ When no match is found, it will retry for seconds (default: 10),
+ so this is also suitable to use as wait condition.
+
+ await :py:obj:`Tab`
+ ---------------------------
+ Calling `await tab` will do a lot of stuff under the hood,
+ and ensures all references are up to date.
+ Also it allows for the script to "breathe",
+ as it is oftentime faster than your browser or webpage.
+ So whenever you get stuck and things crashes or element could not be found,
+ you should probably let it "breathe" by calling `await page`
+ and/or `await page.sleep()`.
+
+ It ensures :py:obj:`~url` will be updated to the most recent one,
+ which is quite important in some other methods.
+
+ Using other and custom CDP commands
+ ======================================================
+ Using the included cdp module, you can easily craft commands,
+ which will always return an generator object.
+ This generator object can be easily sent to the :py:meth:`~send` method.
+
+ :py:meth:`~send`
+ ---------------------------
+ This is probably the most important method,
+ although you won't ever call it, unless you want to go really custom.
+ The send method accepts a :py:obj:`cdp` command.
+ Each of which can be found in the cdp section.
+
+ When you import * from this package, cdp will be in your namespace,
+ and contains all domains/actions/events you can act upon.
+ """
+ browser: cdp_browser.Browser
+ _download_behavior: List[str] = None
+
+ def __init__(
+ self,
+ websocket_url: str,
+ target: cdp.target.TargetInfo,
+ browser: Optional["cdp_browser.Browser"] = None,
+ **kwargs,
+ ):
+ super().__init__(websocket_url, target, browser, **kwargs)
+ self.browser = browser
+ self._dom = None
+ self._window_id = None
+
+ @property
+ def inspector_url(self):
+ """
+ Get the inspector url.
+ This url can be used in another browser to show you
+ the devtools interface for current tab.
+ Useful for debugging and headless mode.
+ """
+ return f"http://{self.browser.config.host}:{self.browser.config.port}/devtools/inspector.html?ws={self.websocket_url[5:]}" # noqa
+
+ def inspector_open(self):
+ import webbrowser
+
+ webbrowser.open(self.inspector_url, new=2)
+
+ async def open_external_inspector(self):
+ """
+ Opens the system's browser containing the devtools inspector page
+ for this tab. Could be handy, especially to debug in headless mode.
+ """
+ import webbrowser
+
+ webbrowser.open(self.inspector_url)
+
+ async def find(
+ self,
+ text: str,
+ best_match: bool = False,
+ return_enclosing_element: bool = True,
+ timeout: Union[int, float] = 10,
+ ):
+ """
+ Find single element by text.
+ Can also be used to wait for such element to appear.
+ :param text:
+ Text to search for. Note: Script contents are also considered text.
+ :type text: str
+ :param best_match: :param best_match:
+ When True (default), it will return the element which has the most
+ comparable string length. This could help a lot. Eg:
+ If you search for "login", you probably want the login button element,
+ and not thousands of tags/scripts containing a "login" string.
+ When False, it returns just the first match (but is way faster).
+ :type best_match: bool
+ :param return_enclosing_element:
+ Since we deal with nodes instead of elements,
+ the find function most often returns so called text nodes,
+ which is actually a element of plain text,
+ which is the somehow imaginary "child" of a "span", "p", "script"
+ or any other elements which have text between their opening
+ and closing tags.
+ Most often when we search by text, we actually aim for the
+ element containing the text instead of a lousy plain text node,
+ so by default the containing element is returned.
+ There are exceptions. Eg:
+ Elements that use the "placeholder=" property.
+ :type return_enclosing_element: bool
+ :param timeout:
+ Raise timeout exception when after this many seconds nothing is found.
+ :type timeout: float,int
+ """
+ loop = asyncio.get_running_loop()
+ start_time = loop.time()
+ text = text.strip()
+ item = None
+ try:
+ item = await self.find_element_by_text(
+ text, best_match, return_enclosing_element
+ )
+ except (Exception, TypeError):
+ pass
+ while not item:
+ await self
+ item = await self.find_element_by_text(
+ text, best_match, return_enclosing_element
+ )
+ if loop.time() - start_time > timeout:
+ raise asyncio.TimeoutError(
+ "Time ran out while waiting for: {%s}" % text
+ )
+ await self.sleep(0.5)
+ return item
+
+ async def select(
+ self,
+ selector: str,
+ timeout: Union[int, float] = 10,
+ ) -> element.Element:
+ """
+ Find a single element by css selector.
+ Can also be used to wait for such an element to appear.
+ :param selector: css selector,
+ eg a[href], button[class*=close], a > img[src]
+ :type selector: str
+ :param timeout:
+ Raise timeout exception when after this many seconds nothing is found.
+ :type timeout: float,int
+ """
+ loop = asyncio.get_running_loop()
+ start_time = loop.time()
+ selector = selector.strip()
+ item = None
+ try:
+ item = await self.query_selector(selector)
+ except (Exception, TypeError):
+ pass
+ while not item:
+ await self
+ item = await self.query_selector(selector)
+ if loop.time() - start_time > timeout:
+ raise asyncio.TimeoutError(
+ "Time ran out while waiting for: {%s}" % selector
+ )
+ await self.sleep(0.5)
+ return item
+
+ async def find_all(
+ self,
+ text: str,
+ timeout: Union[int, float] = 10,
+ ) -> List[element.Element]:
+ """
+ Find multiple elements by text.
+ Can also be used to wait for such elements to appear.
+ :param text: Text to search for.
+ Note: Script contents are also considered text.
+ :type text: str
+ :param timeout:
+ Raise timeout exception when after this many seconds nothing is found.
+ :type timeout: float,int
+ """
+ loop = asyncio.get_running_loop()
+ now = loop.time()
+ text = text.strip()
+ items = []
+ try:
+ items = await self.find_elements_by_text(text)
+ except (Exception, TypeError):
+ pass
+ while not items:
+ await self
+ items = await self.find_elements_by_text(text)
+ if loop.time() - now > timeout:
+ raise asyncio.TimeoutError(
+ "Time ran out while waiting for: {%s}" % text
+ )
+ await self.sleep(0.5)
+ return items
+
+ async def select_all(
+ self,
+ selector: str,
+ timeout: Union[int, float] = 10,
+ include_frames=False,
+ ) -> List[element.Element]:
+ """
+ Find multiple elements by CSS Selector.
+ Can also be used to wait for such elements to appear.
+ :param selector: css selector,
+ eg a[href], button[class*=close], a > img[src]
+ :type selector: str
+ :param timeout:
+ Raise timeout exception when after this many seconds nothing is found.
+ :type timeout: float,int
+ :param include_frames: Whether to include results in iframes.
+ :type include_frames: bool
+ """
+ loop = asyncio.get_running_loop()
+ now = loop.time()
+ selector = selector.strip()
+ items = []
+ if include_frames:
+ frames = await self.query_selector_all("iframe")
+ # Unfortunately, asyncio.gather is not an option here
+ for fr in frames:
+ items.extend(await fr.query_selector_all(selector))
+ items.extend(await self.query_selector_all(selector))
+ while not items:
+ await self
+ items = await self.query_selector_all(selector)
+ if loop.time() - now > timeout:
+ raise asyncio.TimeoutError(
+ "Time ran out while waiting for: {%s}" % selector
+ )
+ await self.sleep(0.5)
+ return items
+
+ async def get(
+ self,
+ url="chrome://welcome",
+ new_tab: bool = False,
+ new_window: bool = False,
+ ):
+ """
+ Top level get. Utilizes the first tab to retrieve the given url.
+ This is a convenience function known from selenium.
+ This function handles waits/sleeps and detects when DOM events fired,
+ so it's the safest way of navigating.
+ :param url: the url to navigate to
+ :param new_tab: open new tab
+ :param new_window: open new window
+ :return: Page
+ """
+ if not self.browser:
+ raise AttributeError(
+ "This page/tab has no browser attribute, "
+ "so you can't use get()"
+ )
+ if new_window and not new_tab:
+ new_tab = True
+ if new_tab:
+ return await self.browser.get(url, new_tab, new_window)
+ else:
+ frame_id, loader_id, *_ = await self.send(cdp.page.navigate(url))
+ await self
+ return self
+
+ async def query_selector_all(
+ self,
+ selector: str,
+ _node: Optional[Union[cdp.dom.Node, "element.Element"]] = None,
+ ):
+ """
+ Equivalent of JavaScript "document.querySelectorAll".
+ This is considered one of the main methods to use in this package.
+ It returns all matching :py:obj:`element.Element` objects.
+ :param selector: css selector.
+ (first time? => https://www.w3schools.com/cssref/css_selectors.php )
+ :type selector: str
+ :param _node: internal use
+ """
+ if not _node:
+ doc: cdp.dom.Node = await self.send(cdp.dom.get_document(-1, True))
+ else:
+ doc = _node
+ if _node.node_name == "IFRAME":
+ doc = _node.content_document
+ node_ids = []
+ try:
+ node_ids = await self.send(
+ cdp.dom.query_selector_all(doc.node_id, selector)
+ )
+ except ProtocolException as e:
+ if _node is not None:
+ if "could not find node" in e.message.lower():
+ if getattr(_node, "__last", None):
+ del _node.__last
+ return []
+ # If the supplied node is not found,
+ # then the DOM has changed since acquiring the element.
+ # Therefore, we need to update our node, and try again.
+ await _node.update()
+ _node.__last = (
+ True # Make sure this isn't turned into infinite loop.
+ )
+ return await self.query_selector_all(selector, _node)
+ else:
+ await self.send(cdp.dom.disable())
+ raise
+ if not node_ids:
+ return []
+ items = []
+ for nid in node_ids:
+ node = util.filter_recurse(doc, lambda n: n.node_id == nid)
+ # Pass along the retrieved document tree to improve performance.
+ if not node:
+ continue
+ elem = element.create(node, self, doc)
+ items.append(elem)
+ return items
+
+ async def query_selector(
+ self,
+ selector: str,
+ _node: Optional[Union[cdp.dom.Node, element.Element]] = None,
+ ):
+ """
+ Find a single element based on a CSS Selector string.
+ :param selector: CSS Selector(s)
+ :type selector: str
+ """
+ selector = selector.strip()
+ if not _node:
+ doc: cdp.dom.Node = await self.send(cdp.dom.get_document(-1, True))
+ else:
+ doc = _node
+ if _node.node_name == "IFRAME":
+ doc = _node.content_document
+ node_id = None
+ try:
+ node_id = await self.send(
+ cdp.dom.query_selector(doc.node_id, selector)
+ )
+ except ProtocolException as e:
+ if _node is not None:
+ if "could not find node" in e.message.lower():
+ if getattr(_node, "__last", None):
+ del _node.__last
+ return []
+ # If supplied node is not found,
+ # the dom has changed since acquiring the element,
+ # therefore, update our passed node and try again.
+ await _node.update()
+ _node.__last = (
+ True # Make sure this isn't turned into infinite loop.
+ )
+ return await self.query_selector(selector, _node)
+ else:
+ await self.send(cdp.dom.disable())
+ raise
+ if not node_id:
+ return
+ node = util.filter_recurse(doc, lambda n: n.node_id == node_id)
+ if not node:
+ return
+ return element.create(node, self, doc)
+
+ async def find_elements_by_text(
+ self,
+ text: str,
+ ) -> List[element.Element]:
+ """
+ Returns element which match the given text.
+ Note: This may (or will) also return any other element
+ (like inline scripts), which happen to contain that text.
+ :param text:
+ """
+ text = text.strip()
+ doc = await self.send(cdp.dom.get_document(-1, True))
+ search_id, nresult = await self.send(
+ cdp.dom.perform_search(text, True)
+ )
+ if nresult:
+ node_ids = await self.send(
+ cdp.dom.get_search_results(search_id, 0, nresult)
+ )
+ else:
+ node_ids = []
+ await self.send(cdp.dom.discard_search_results(search_id))
+ items = []
+ for nid in node_ids:
+ node = util.filter_recurse(doc, lambda n: n.node_id == nid)
+ if not node:
+ node = await self.send(cdp.dom.resolve_node(node_id=nid))
+ if not node:
+ continue
+ # remote_object = await self.send(
+ # cdp.dom.resolve_node(backend_node_id=node.backend_node_id)
+ # )
+ # node_id = await self.send(
+ # cdp.dom.request_node(object_id=remote_object.object_id)
+ # )
+ try:
+ elem = element.create(node, self, doc)
+ except BaseException:
+ continue
+ if elem.node_type == 3:
+ # If found element is a text node (which is plain text,
+ # and useless for our purpose), we return the parent element
+ # of the node (which is often a tag which can have text
+ # between their opening and closing tags (that is most tags,
+ # except for example "img" and "video", "br").
+ if not elem.parent:
+ # Check if parent actually has a parent
+ # and update it to be absolutely sure.
+ await elem.update()
+ items.append(
+ elem.parent or elem
+ ) # When there's no parent, use the text node itself.
+ continue
+ else:
+ # Add the element itself.
+ items.append(elem)
+ # Since we already fetched the entire doc, including shadow and frames,
+ # let's also search through the iframes.
+ iframes = util.filter_recurse_all(
+ doc, lambda node: node.node_name == "IFRAME"
+ )
+ if iframes:
+ iframes_elems = [
+ element.create(iframe, self, iframe.content_document)
+ for iframe in iframes
+ ]
+ for iframe_elem in iframes_elems:
+ if iframe_elem.content_document:
+ iframe_text_nodes = util.filter_recurse_all(
+ iframe_elem,
+ lambda node: node.node_type == 3 # noqa
+ and text.lower() in node.node_value.lower(),
+ )
+ if iframe_text_nodes:
+ iframe_text_elems = [
+ element.create(text_node, self, iframe_elem.tree)
+ for text_node in iframe_text_nodes
+ ]
+ items.extend(
+ text_node.parent for text_node in iframe_text_elems
+ )
+ await self.send(cdp.dom.disable())
+ return items or []
+
+ async def find_element_by_text(
+ self,
+ text: str,
+ best_match: Optional[bool] = False,
+ return_enclosing_element: Optional[bool] = True,
+ ) -> Union[element.Element, None]:
+ """
+ Finds and returns the first element containing , or best match.
+ :param text:
+ :param best_match:
+ When True, which is MUCH more expensive (thus much slower),
+ will find the closest match based on length.
+ When searching for "login", you probably want the button element,
+ and not thousands of tags/scripts containing the "login" string.
+ :type best_match: bool
+ :param return_enclosing_element:
+ """
+ doc = await self.send(cdp.dom.get_document(-1, True))
+ text = text.strip()
+ search_id, nresult = await self.send(
+ cdp.dom.perform_search(text, True)
+ )
+ node_ids = await self.send(
+ cdp.dom.get_search_results(search_id, 0, nresult)
+ )
+ await self.send(cdp.dom.discard_search_results(search_id))
+ if not node_ids:
+ node_ids = []
+ items = []
+ for nid in node_ids:
+ node = util.filter_recurse(doc, lambda n: n.node_id == nid)
+ try:
+ elem = element.create(node, self, doc)
+ except BaseException:
+ continue
+ if elem.node_type == 3:
+ # If found element is a text node
+ # (which is plain text, and useless for our purpose),
+ # then return the parent element of the node
+ # (which is often a tag which can have text between their
+ # opening and closing tags (that is most tags,
+ # except for example "img" and "video", "br").
+ if not elem.parent:
+ # Check if parent has a parent, and update it to be sure.
+ await elem.update()
+ items.append(
+ elem.parent or elem
+ ) # When it really has no parent, use the text node itself
+ continue
+ else:
+ # Add the element itself
+ items.append(elem)
+ # Since the entire doc is already fetched, including shadow and frames,
+ # also search through the iframes.
+ iframes = util.filter_recurse_all(
+ doc, lambda node: node.node_name == "IFRAME"
+ )
+ if iframes:
+ iframes_elems = [
+ element.create(iframe, self, iframe.content_document)
+ for iframe in iframes
+ ]
+ for iframe_elem in iframes_elems:
+ iframe_text_nodes = util.filter_recurse_all(
+ iframe_elem,
+ lambda node: node.node_type == 3 # noqa
+ and text.lower() in node.node_value.lower(),
+ )
+ if iframe_text_nodes:
+ iframe_text_elems = [
+ element.create(text_node, self, iframe_elem.tree)
+ for text_node in iframe_text_nodes
+ ]
+ items.extend(
+ text_node.parent for text_node in iframe_text_elems
+ )
+ try:
+ if not items:
+ return
+ if best_match:
+ closest_by_length = min(
+ items, key=lambda el: abs(len(text) - len(el.text_all))
+ )
+ elem = closest_by_length or items[0]
+ return elem
+ else:
+ # Return the first result
+ for elem in items:
+ if elem:
+ return elem
+ finally:
+ await self.send(cdp.dom.disable())
+
+ async def back(self):
+ """History back"""
+ await self.send(cdp.runtime.evaluate("window.history.back()"))
+
+ async def forward(self):
+ """History forward"""
+ await self.send(cdp.runtime.evaluate("window.history.forward()"))
+
+ async def reload(
+ self,
+ ignore_cache: Optional[bool] = True,
+ script_to_evaluate_on_load: Optional[str] = None,
+ ):
+ """
+ Reloads the page
+ :param ignore_cache: When set to True (default),
+ it ignores cache, and re-downloads the items.
+ :param script_to_evaluate_on_load: Script to run on load.
+ """
+ await self.send(
+ cdp.page.reload(
+ ignore_cache=ignore_cache,
+ script_to_evaluate_on_load=script_to_evaluate_on_load,
+ ),
+ )
+
+ async def evaluate(
+ self, expression: str, await_promise=False, return_by_value=True
+ ):
+ remote_object, errors = await self.send(
+ cdp.runtime.evaluate(
+ expression=expression,
+ user_gesture=True,
+ await_promise=await_promise,
+ return_by_value=return_by_value,
+ allow_unsafe_eval_blocked_by_csp=True,
+ )
+ )
+ if errors:
+ raise ProtocolException(errors)
+ if remote_object:
+ if return_by_value:
+ if remote_object.value:
+ return remote_object.value
+ else:
+ return remote_object, errors
+
+ async def js_dumps(
+ self, obj_name: str, return_by_value: Optional[bool] = True
+ ) -> Union[
+ Dict,
+ Tuple[cdp.runtime.RemoteObject, cdp.runtime.ExceptionDetails],
+ ]:
+ """
+ Dump Given js object with its properties and values as a dict.
+ Note: Complex objects might not be serializable,
+ therefore this method is not a "source of truth"
+ :param obj_name: the js object to dump
+ :type obj_name: str
+ :param return_by_value: If you want an tuple of cdp objects
+ (returnvalue, errors), then set this to False.
+ :type return_by_value: bool
+
+ Example
+ -------
+
+ x = await self.js_dumps('window')
+ print(x)
+ '...{
+ 'pageYOffset': 0,
+ 'visualViewport': {},
+ 'screenX': 10,
+ 'screenY': 10,
+ 'outerWidth': 1050,
+ 'outerHeight': 832,
+ 'devicePixelRatio': 1,
+ 'screenLeft': 10,
+ 'screenTop': 10,
+ 'styleMedia': {},
+ 'onsearch': None,
+ 'isSecureContext': True,
+ 'trustedTypes': {},
+ 'performance': {'timeOrigin': 1707823094767.9,
+ 'timing': {'connectStart': 0,
+ 'navigationStart': 1707823094768,
+ ]...
+ """
+ js_code_a = (
+ """
+ function ___dump(obj, _d = 0) {
+ let _typesA = ['object', 'function'];
+ let _typesB = ['number', 'string', 'boolean'];
+ if (_d == 2) {
+ console.log('maxdepth reached for ', obj);
+ return
+ }
+ let tmp = {}
+ for (let k in obj) {
+ if (obj[k] == window) continue;
+ let v;
+ try {
+ if (obj[k] === null
+ || obj[k] === undefined
+ || obj[k] === NaN) {
+ console.log('obj[k] is null or undefined or Nan',
+ k, '=>', obj[k])
+ tmp[k] = obj[k];
+ continue
+ }
+ } catch (e) {
+ tmp[k] = null;
+ continue
+ }
+ if (_typesB.includes(typeof obj[k])) {
+ tmp[k] = obj[k]
+ continue
+ }
+ try {
+ if (typeof obj[k] === 'function') {
+ tmp[k] = obj[k].toString()
+ continue
+ }
+ if (typeof obj[k] === 'object') {
+ tmp[k] = ___dump(obj[k], _d + 1);
+ continue
+ }
+ } catch (e) {}
+ try {
+ tmp[k] = JSON.stringify(obj[k])
+ continue
+ } catch (e) {
+ }
+ try {
+ tmp[k] = obj[k].toString();
+ continue
+ } catch (e) {}
+ }
+ return tmp
+ }
+ function ___dumpY(obj) {
+ var objKeys = (obj) => {
+ var [target, result] = [obj, []];
+ while (target !== null) {
+ result = result.concat(
+ Object.getOwnPropertyNames(target)
+ );
+ target = Object.getPrototypeOf(target);
+ }
+ return result;
+ }
+ return Object.fromEntries(
+ objKeys(obj).map(_ => [_, ___dump(obj[_])]))
+ }
+ ___dumpY( %s )
+ """
+ % obj_name
+ )
+ js_code_b = (
+ """
+ ((obj, visited = new WeakSet()) => {
+ if (visited.has(obj)) {
+ return {}
+ }
+ visited.add(obj)
+ var result = {}, _tmp;
+ for (var i in obj) {
+ try {
+ if (i === 'enabledPlugin'
+ || typeof obj[i] === 'function') {
+ continue;
+ } else if (typeof obj[i] === 'object') {
+ _tmp = recurse(obj[i], visited);
+ if (Object.keys(_tmp).length) {
+ result[i] = _tmp;
+ }
+ } else {
+ result[i] = obj[i];
+ }
+ } catch (error) {
+ // console.error('Error:', error);
+ }
+ }
+ return result;
+ })(%s)
+ """
+ % obj_name
+ )
+ # No self.evaluate here to prevent infinite loop on certain expressions
+ remote_object, exception_details = await self.send(
+ cdp.runtime.evaluate(
+ js_code_a,
+ await_promise=True,
+ return_by_value=return_by_value,
+ allow_unsafe_eval_blocked_by_csp=True,
+ )
+ )
+ if exception_details:
+ # Try second variant
+ remote_object, exception_details = await self.send(
+ cdp.runtime.evaluate(
+ js_code_b,
+ await_promise=True,
+ return_by_value=return_by_value,
+ allow_unsafe_eval_blocked_by_csp=True,
+ )
+ )
+ if exception_details:
+ raise ProtocolException(exception_details)
+ if return_by_value:
+ if remote_object.value:
+ return remote_object.value
+ else:
+ return remote_object, exception_details
+
+ async def close(self):
+ """Close the current target (ie: tab,window,page)"""
+ if self.target and self.target.target_id:
+ await self.send(
+ cdp.target.close_target(target_id=self.target.target_id)
+ )
+
+ async def get_window(self) -> Tuple[
+ cdp.browser.WindowID, cdp.browser.Bounds
+ ]:
+ """Get the window Bounds"""
+ window_id, bounds = await self.send(
+ cdp.browser.get_window_for_target(self.target_id)
+ )
+ return window_id, bounds
+
+ async def get_content(self):
+ """Gets the current page source content (html)"""
+ doc: cdp.dom.Node = await self.send(cdp.dom.get_document(-1, True))
+ return await self.send(
+ cdp.dom.get_outer_html(backend_node_id=doc.backend_node_id)
+ )
+
+ async def maximize(self):
+ """Maximize page/tab/window"""
+ return await self.set_window_state(state="maximize")
+
+ async def minimize(self):
+ """Minimize page/tab/window"""
+ return await self.set_window_state(state="minimize")
+
+ async def fullscreen(self):
+ """Minimize page/tab/window"""
+ return await self.set_window_state(state="fullscreen")
+
+ async def medimize(self):
+ return await self.set_window_state(state="normal")
+
+ async def set_window_size(self, left=0, top=0, width=1280, height=1024):
+ """
+ Set window size and position.
+ :param left:
+ Pixels from the left of the screen to the window top-left corner.
+ :param top:
+ Pixels from the top of the screen to the window top-left corner.
+ :param width: width of the window in pixels
+ :param height: height of the window in pixels
+ """
+ return await self.set_window_state(left, top, width, height)
+
+ async def activate(self):
+ """Active this target (Eg: tab, window, page)"""
+ await self.send(cdp.target.activate_target(self.target.target_id))
+
+ async def bring_to_front(self):
+ """Alias to self.activate"""
+ await self.activate()
+
+ async def set_window_state(
+ self, left=0, top=0, width=1280, height=720, state="normal"
+ ):
+ """
+ Sets the window size or state.
+ For state you can provide the full name like minimized, maximized,
+ normal, fullscreen, or something which leads to either of those,
+ like min, mini, mi, max, ma, maxi, full, fu, no, nor.
+ In case state is set other than "normal",
+ the left, top, width, and height are ignored.
+ :param left:
+ desired offset from left, in pixels
+ :type left: int
+ :param top:
+ desired offset from the top, in pixels
+ :type top: int
+ :param width:
+ desired width in pixels
+ :type width: int
+ :param height:
+ desired height in pixels
+ :type height: int
+ :param state:
+ can be one of the following strings:
+ - normal
+ - fullscreen
+ - maximized
+ - minimized
+ :type state: str
+ """
+ available_states = ["minimized", "maximized", "fullscreen", "normal"]
+ window_id: cdp.browser.WindowID
+ bounds: cdp.browser.Bounds
+ (window_id, bounds) = await self.get_window()
+ for state_name in available_states:
+ if all(x in state_name for x in state.lower()):
+ break
+ else:
+ raise NameError(
+ "could not determine any of %s from input '%s'"
+ % (",".join(available_states), state)
+ )
+ window_state = getattr(
+ cdp.browser.WindowState,
+ state_name.upper(),
+ cdp.browser.WindowState.NORMAL,
+ )
+ if window_state == cdp.browser.WindowState.NORMAL:
+ bounds = cdp.browser.Bounds(
+ left, top, width, height, window_state
+ )
+ else:
+ # min, max, full can only be used when current state == NORMAL,
+ # therefore, first switch to NORMAL
+ await self.set_window_state(state="normal")
+ bounds = cdp.browser.Bounds(window_state=window_state)
+
+ await self.send(
+ cdp.browser.set_window_bounds(window_id, bounds=bounds)
+ )
+
+ async def scroll_down(self, amount=25):
+ """
+ Scrolls the page down.
+ :param amount: Number in percentage.
+ 25 is a quarter of page, 50 half, and 1000 is 10x the page.
+ :type amount: int
+ """
+ window_id: cdp.browser.WindowID
+ bounds: cdp.browser.Bounds
+ (window_id, bounds) = await self.get_window()
+ await self.send(
+ cdp.input_.synthesize_scroll_gesture(
+ x=0,
+ y=0,
+ y_distance=-(bounds.height * (amount / 100)),
+ y_overscroll=0,
+ x_overscroll=0,
+ prevent_fling=True,
+ repeat_delay_ms=0,
+ speed=7777,
+ )
+ )
+
+ async def scroll_up(self, amount=25):
+ """
+ Scrolls the page up.
+ :param amount: Number in percentage.
+ 25 is a quarter of page, 50 half, and 1000 is 10x the page.
+ :type amount: int
+ """
+ window_id: cdp.browser.WindowID
+ bounds: cdp.browser.Bounds
+ (window_id, bounds) = await self.get_window()
+ await self.send(
+ cdp.input_.synthesize_scroll_gesture(
+ x=0,
+ y=0,
+ y_distance=(bounds.height * (amount / 100)),
+ x_overscroll=0,
+ prevent_fling=True,
+ repeat_delay_ms=0,
+ speed=7777,
+ )
+ )
+
+ async def wait_for(
+ self,
+ selector: Optional[str] = "",
+ text: Optional[str] = "",
+ timeout: Optional[Union[int, float]] = 10,
+ ) -> element.Element:
+ """
+ Variant on query_selector_all and find_elements_by_text.
+ This variant takes either selector or text,
+ and will block until the requested element(s) are found.
+ It will block for a maximum of seconds,
+ after which a TimeoutError will be raised.
+ :param selector: css selector
+ :param text: text
+ :param timeout:
+ :return: Element
+ :raises: asyncio.TimeoutError
+ """
+ loop = asyncio.get_running_loop()
+ now = loop.time()
+ if selector:
+ item = await self.query_selector(selector)
+ while not item:
+ item = await self.query_selector(selector)
+ if loop.time() - now > timeout:
+ raise asyncio.TimeoutError(
+ "Time ran out while waiting for: {%s}" % selector
+ )
+ await self.sleep(0.5)
+ return item
+ if text:
+ item = await self.find_element_by_text(text)
+ while not item:
+ item = await self.find_element_by_text(text)
+ if loop.time() - now > timeout:
+ raise asyncio.TimeoutError(
+ "Time ran out while waiting for: {%s}" % text
+ )
+ await self.sleep(0.5)
+ return item
+
+ async def download_file(
+ self, url: str, filename: Optional[PathLike] = None
+ ):
+ """
+ Downloads the file by the given url.
+ :param url: The URL of the file.
+ :param filename: The name for the file.
+ If not specified, the name is composed from the url file name
+ """
+ if not self._download_behavior:
+ directory_path = pathlib.Path.cwd() / "downloads"
+ directory_path.mkdir(exist_ok=True)
+ await self.set_download_path(directory_path)
+
+ warnings.warn(
+ f"No download path set, so creating and using a default of "
+ f"{directory_path}"
+ )
+ if not filename:
+ filename = url.rsplit("/")[-1]
+ filename = filename.split("?")[0]
+ code = """
+ (elem) => {
+ async function _downloadFile(
+ imageSrc,
+ nameOfDownload,
+ ) {
+ const response = await fetch(imageSrc);
+ const blobImage = await response.blob();
+ const href = URL.createObjectURL(blobImage);
+ const anchorElement = document.createElement('a');
+ anchorElement.href = href;
+ anchorElement.download = nameOfDownload;
+ document.body.appendChild(anchorElement);
+ anchorElement.click();
+ setTimeout(() => {
+ document.body.removeChild(anchorElement);
+ window.URL.revokeObjectURL(href);
+ }, 500);
+ }
+ _downloadFile('%s', '%s')
+ }
+ """ % (
+ url,
+ filename,
+ )
+ body = (await self.query_selector_all("body"))[0]
+ await body.update()
+ await self.send(
+ cdp.runtime.call_function_on(
+ code,
+ object_id=body.object_id,
+ arguments=[cdp.runtime.CallArgument(object_id=body.object_id)],
+ )
+ )
+
+ async def save_screenshot(
+ self,
+ filename: Optional[PathLike] = "auto",
+ format: Optional[str] = "png",
+ full_page: Optional[bool] = False,
+ ) -> str:
+ """
+ Saves a screenshot of the page.
+ This is not the same as :py:obj:`Element.save_screenshot`,
+ which saves a screenshot of a single element only.
+ :param filename: uses this as the save path
+ :type filename: PathLike
+ :param format: jpeg or png (defaults to jpeg)
+ :type format: str
+ :param full_page:
+ When False (default), it captures the current viewport.
+ When True, it captures the entire page.
+ :type full_page: bool
+ :return: The path/filename of the saved screenshot.
+ :rtype: str
+ """
+ import urllib.parse
+ import datetime
+
+ await self.sleep() # Update the target's URL
+ path = None
+ if format.lower() in ["jpg", "jpeg"]:
+ ext = ".jpg"
+ format = "jpeg"
+ elif format.lower() in ["png"]:
+ ext = ".png"
+ format = "png"
+ if not filename or filename == "auto":
+ parsed = urllib.parse.urlparse(self.target.url)
+ parts = parsed.path.split("/")
+ last_part = parts[-1]
+ last_part = last_part.rsplit("?", 1)[0]
+ dt_str = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+ candidate = f"{parsed.hostname}__{last_part}_{dt_str}"
+ path = pathlib.Path(candidate + ext) # noqa
+ else:
+ path = pathlib.Path(filename)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ data = await self.send(
+ cdp.page.capture_screenshot(
+ format_=format, capture_beyond_viewport=full_page
+ )
+ )
+ if not data:
+ raise ProtocolException(
+ "Could not take screenshot. "
+ "Most possible cause is the page "
+ "has not finished loading yet."
+ )
+ import base64
+
+ data_bytes = base64.b64decode(data)
+ if not path:
+ raise RuntimeError("Invalid filename or path: '%s'" % filename)
+ path.write_bytes(data_bytes)
+ return str(path)
+
+ async def set_download_path(self, path: PathLike):
+ """
+ Sets the download path.
+ When not set, a default folder is used.
+ :param path:
+ """
+ await self.send(
+ cdp.browser.set_download_behavior(
+ behavior="allow", download_path=str(path.resolve())
+ )
+ )
+ self._download_behavior = ["allow", str(path.resolve())]
+
+ async def get_all_linked_sources(self) -> List["element.Element"]:
+ """Get all elements of tag: link, a, img, scripts meta, video, audio"""
+ all_assets = await self.query_selector_all(
+ selector="a,link,img,script,meta"
+ )
+ return [element.create(asset, self) for asset in all_assets]
+
+ async def get_all_urls(self, absolute=True) -> List[str]:
+ """
+ Convenience function, which returns all links (a,link,img,script,meta).
+ :param absolute:
+ Try to build all the links in absolute form
+ instead of "as is", often relative.
+ :return: List of URLs.
+ """
+ import urllib.parse
+
+ res = []
+ all_assets = await self.query_selector_all(
+ selector="a,link,img,script,meta"
+ )
+ for asset in all_assets:
+ if not absolute:
+ res.append(asset.src or asset.href)
+ else:
+ for k, v in asset.attrs.items():
+ if k in ("src", "href"):
+ if "#" in v:
+ continue
+ if not any([_ in v for _ in ("http", "//", "/")]):
+ continue
+ abs_url = urllib.parse.urljoin(
+ "/".join(self.url.rsplit("/")[:3]), v
+ )
+ if not abs_url.startswith(("http", "//", "ws")):
+ continue
+ res.append(abs_url)
+ return res
+
+ async def verify_cf(self):
+ """(An attempt)"""
+ checkbox = None
+ checkbox_sibling = await self.wait_for(text="verify you are human")
+ if checkbox_sibling:
+ parent = checkbox_sibling.parent
+ while parent:
+ checkbox = await parent.query_selector("input[type=checkbox]")
+ if checkbox:
+ break
+ parent = parent.parent
+ await checkbox.mouse_move()
+ await checkbox.mouse_click()
+
+ async def get_document(self):
+ return await self.send(cdp.dom.get_document())
+
+ async def get_flattened_document(self):
+ return await self.send(cdp.dom.get_flattened_document())
+
+ async def get_local_storage(self):
+ """
+ Get local storage items as dict of strings.
+ Proper deserialization may need to be done.
+ """
+ if not self.target.url:
+ await self
+ origin = "/".join(self.url.split("/", 3)[:-1])
+ items = await self.send(
+ cdp.dom_storage.get_dom_storage_items(
+ cdp.dom_storage.StorageId(
+ is_local_storage=True, security_origin=origin
+ )
+ )
+ )
+ retval = {}
+ for item in items:
+ retval[item[0]] = item[1]
+ return retval
+
+ async def set_local_storage(self, items: dict):
+ """
+ Set local storage.
+ Dict items must be strings.
+ Simple types will be converted to strings automatically.
+ :param items: dict containing {key:str, value:str}
+ :type items: dict[str,str]
+ """
+ if not self.target.url:
+ await self
+ origin = "/".join(self.url.split("/", 3)[:-1])
+ await asyncio.gather(
+ *[
+ self.send(
+ cdp.dom_storage.set_dom_storage_item(
+ storage_id=cdp.dom_storage.StorageId(
+ is_local_storage=True, security_origin=origin
+ ),
+ key=str(key),
+ value=str(val),
+ )
+ )
+ for key, val in items.items()
+ ]
+ )
+
+ def __call__(
+ self,
+ text: Optional[str] = "",
+ selector: Optional[str] = "",
+ timeout: Optional[Union[int, float]] = 10,
+ ):
+ """
+ Alias to query_selector_all or find_elements_by_text,
+ depending on whether text= is set or selector= is set.
+ :param selector: css selector string
+ :type selector: str
+ """
+ return self.wait_for(text, selector, timeout)
+
+ def __eq__(self, other: Tab):
+ try:
+ return other.target == self.target
+ except (AttributeError, TypeError):
+ return False
+
+ def __getattr__(self, item):
+ try:
+ return getattr(self._target, item)
+ except AttributeError:
+ raise AttributeError(
+ f'"{self.__class__.__name__}" has no attribute "%s"' % item
+ )
+
+ def __repr__(self):
+ extra = ""
+ if self.target.url:
+ extra = f"[url: {self.target.url}]"
+ s = f"<{type(self).__name__} [{self.target_id}] [{self.type_}] {extra}>" # noqa
+ return s
diff --git a/seleniumbase/undetected/patcher.py b/seleniumbase/undetected/patcher.py
index 1375e177b61..08c775cb089 100644
--- a/seleniumbase/undetected/patcher.py
+++ b/seleniumbase/undetected/patcher.py
@@ -165,9 +165,9 @@ def unzip_package(self, fp):
@staticmethod
def force_kill_instances(exe_name):
- """ Terminate instances of UC.
- :param: executable name to kill, may be a path as well
- :return: True on success else False """
+ """Terminate instances of UC.
+ :param: Executable name to kill. (Can be a path)
+ :return: True on success else False."""
exe_name = os.path.basename(exe_name)
if IS_POSIX:
r = os.system("kill -f -9 $(pidof %s)" % exe_name)
@@ -274,8 +274,8 @@ def __repr__(self):
def __del__(self):
if self._custom_exe_path:
- # if the driver binary is specified by user
- # we assume it is important enough to not delete it
+ # If the driver binary is specified by the user,
+ # then assume it is important enough to keep it.
return
else:
timeout = 3
diff --git a/setup.py b/setup.py
index b25d804f505..969e32c4ed0 100755
--- a/setup.py
+++ b/setup.py
@@ -147,16 +147,18 @@
],
python_requires=">=3.8",
install_requires=[
- 'pip>=24.1.2', # 24.2: editable install warnings
+ 'pip>=24.2',
'packaging>=24.1',
- 'setuptools~=70.2;python_version<"3.10"',
- 'setuptools>=70.2.0;python_version>="3.10"', # 71.0.x has issues
+ 'setuptools~=70.2;python_version<"3.10"', # Newer ones had issues
+ 'setuptools>=73.0.1;python_version>="3.10"',
'wheel>=0.44.0',
'attrs>=24.2.0',
"certifi>=2024.8.30",
"exceptiongroup>=1.2.2",
+ "websockets>=13.1",
'filelock>=3.16.1',
'fasteners>=0.19',
+ "mycdp>=1.0.1",
"pynose>=1.5.3",
'platformdirs>=4.3.6',
'typing-extensions>=4.12.2',
@@ -172,14 +174,14 @@
"pdbp>=1.5.4",
"idna==3.10",
'chardet==5.2.0',
- 'charset-normalizer==3.3.2',
+ 'charset-normalizer==3.4.0',
'urllib3>=1.26.20,<2;python_version<"3.10"',
'urllib3>=1.26.20,<2.3.0;python_version>="3.10"',
- 'requests==2.31.0',
+ 'requests==2.32.3',
'sniffio==1.3.1',
'h11==0.14.0',
'outcome==1.3.0.post0',
- 'trio==0.26.2',
+ 'trio==0.27.0',
'trio-websocket==0.11.1',
'wsproto==1.2.0',
'websocket-client==1.8.0',
@@ -204,7 +206,7 @@
'python-xlib==0.33;platform_system=="Linux"',
'markdown-it-py==3.0.0',
'mdurl==0.1.2',
- 'rich==13.9.2',
+ 'rich==13.9.3',
],
extras_require={
# pip install -e .[allure]
@@ -218,7 +220,8 @@
# pip install -e .[coverage]
# Usage: coverage run -m pytest; coverage html; coverage report
"coverage": [
- 'coverage>=7.6.1',
+ 'coverage>=7.6.1;python_version<"3.9"',
+ 'coverage>=7.6.4;python_version>="3.9"',
'pytest-cov>=5.0.0',
],
# pip install -e .[flake8]
@@ -238,19 +241,25 @@
"ipdb==0.13.13",
'ipython==7.34.0',
],
+ # pip install -e .[mss]
+ # (An optional library for tile_windows() in CDP Mode.)
+ "mss": [
+ "mss==9.0.2", # Next one drops Python 3.8/3.9
+ ],
# pip install -e .[pdfminer]
# (An optional library for parsing PDF files.)
"pdfminer": [
'pdfminer.six==20240706',
'cryptography==39.0.2;python_version<"3.9"',
- 'cryptography==43.0.1;python_version>="3.9"',
+ 'cryptography==43.0.3;python_version>="3.9"',
'cffi==1.17.1',
"pycparser==2.22",
],
# pip install -e .[pillow]
# (An optional library for image-processing.)
"pillow": [
- 'Pillow>=10.4.0',
+ 'Pillow>=10.4.0;python_version<"3.9"',
+ 'Pillow>=11.0.0;python_version>="3.9"',
],
# pip install -e .[pip-system-certs]
# (If you see [SSL: CERTIFICATE_VERIFY_FAILED], then get this.)
](https://github.com/seleniumbase/SeleniumBase/) CDP Mode π
+
+π SeleniumBase CDP Mode (Chrome Devtools Protocol Mode) is a special mode inside of SeleniumBase UC Mode that lets bots appear human while controlling the browser with the CDP-Driver. Although regular UC Mode can't perform WebDriver actions while the 
+
+