diff --git a/lib/PuppeteerSharp.Nunit/TestExpectations/TestExpectations.local.json b/lib/PuppeteerSharp.Nunit/TestExpectations/TestExpectations.local.json index f3a7c31df..38951f7a5 100644 --- a/lib/PuppeteerSharp.Nunit/TestExpectations/TestExpectations.local.json +++ b/lib/PuppeteerSharp.Nunit/TestExpectations/TestExpectations.local.json @@ -135,6 +135,21 @@ "FAIL" ] }, + { + "comment": "This is part of organizing the webdriver bidi implementation, We will remove it one by one", + "testIdPattern": "[navigation.spec] *should navigate to dataURL and fire dataURL requests*", + "platforms": [ + "darwin", + "linux", + "win32" + ], + "parameters": [ + "webDriverBiDi" + ], + "expectations": [ + "FAIL" + ] + }, { "comment": "This is part of organizing the webdriver bidi implementation, We will remove it one by one", "testIdPattern": "[navigation.spec] navigation Frame.goto*", @@ -405,6 +420,21 @@ "FAIL" ] }, + { + "comment": "This is part of organizing the webdriver bidi implementation, We will remove it one by one", + "testIdPattern": "[browsercontext.spec] *", + "platforms": [ + "darwin", + "linux", + "win32" + ], + "parameters": [ + "webDriverBiDi" + ], + "expectations": [ + "FAIL" + ] + }, { "comment": "This is part of organizing the webdriver bidi implementation, We will remove it one by one", "testIdPattern": "[device-request-prompt.spec] *", diff --git a/lib/PuppeteerSharp.Nunit/TestExpectations/TestExpectations.upstream.json b/lib/PuppeteerSharp.Nunit/TestExpectations/TestExpectations.upstream.json index f3898292e..5663d7502 100644 --- a/lib/PuppeteerSharp.Nunit/TestExpectations/TestExpectations.upstream.json +++ b/lib/PuppeteerSharp.Nunit/TestExpectations/TestExpectations.upstream.json @@ -1,11 +1,4 @@ [ - { - "testIdPattern": "[ariaqueryhandler.spec] *", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[autofill.spec] *", "platforms": ["darwin", "linux", "win32"], @@ -18,7 +11,7 @@ "platforms": ["darwin", "linux", "win32"], "parameters": ["webDriverBiDi"], "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Chrome-only feature blocked on https://github.com/GoogleChromeLabs/chromium-bidi/issues/2082" }, { "testIdPattern": "[device-request-prompt.spec] *", @@ -70,28 +63,14 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch *", - "platforms": ["win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[network.spec] network Network Events *", + "testIdPattern": "[input.spec] input tests Page.waitForFileChooser *", "platforms": ["darwin", "linux", "win32"], "parameters": ["webDriverBiDi"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "No BiDi spec support for this" }, { - "testIdPattern": "[network.spec] network Page.authenticate *", - "platforms": ["darwin", "linux"], - "parameters": ["webDriverBiDi"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[network.spec] network Page.authenticate *", + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch *", "platforms": ["win32"], "parameters": ["webDriverBiDi"], "expectations": ["SKIP"], @@ -111,13 +90,6 @@ "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[network.spec] network Page.setExtraHTTPHeaders *", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[network.spec] network Request.isNavigationRequest *", "platforms": ["darwin", "linux", "win32"], @@ -140,14 +112,14 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[network.spec] network Response.text *", + "testIdPattern": "[network.spec] network Response.remoteAddress *", "platforms": ["darwin", "linux", "win32"], "parameters": ["webDriverBiDi"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "expectations": ["FAIL"], + "comment": "Bidi does not have equivalent field" }, { - "testIdPattern": "[page.spec] Page Page.Events.Popup *", + "testIdPattern": "[network.spec] network Response.text *", "platforms": ["darwin", "linux", "win32"], "parameters": ["webDriverBiDi"], "expectations": ["SKIP"], @@ -175,25 +147,18 @@ "comment": "Chrome-specific test" }, { - "testIdPattern": "[requestinterception-experimental.spec] *", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[requestinterception.spec] *", + "testIdPattern": "[screencast.spec] *", "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], + "parameters": ["chrome"], "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Currently no reliable ffmpeg downloads for testing https://github.com/puppeteer/puppeteer/issues/12121" }, { "testIdPattern": "[screencast.spec] *", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox"], "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "CDP-specific feature" }, { "testIdPattern": "[screenshot.spec] Screenshots Cdp *", @@ -213,7 +178,7 @@ "testIdPattern": "[accessibility.spec] *", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], - "expectations": ["SKIP", "TIMEOUT"], + "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { @@ -224,60 +189,25 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[autofill.spec] *", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "chrome-headless-shell"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[browsercontext.spec] *", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[browsercontext.spec] BrowserContext BrowserContext.overridePermissions should be prompt by default", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] - }, - { - "testIdPattern": "[browsercontext.spec] BrowserContext should close all belonging targets once closing context", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] - }, - { - "testIdPattern": "[browsercontext.spec] BrowserContext should create new incognito context", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] - }, - { - "testIdPattern": "[browsercontext.spec] BrowserContext should have default context", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] - }, - { - "testIdPattern": "[browsercontext.spec] BrowserContext should timeout waiting for a non-existent target", + "testIdPattern": "[ariaqueryhandler.spec] *", "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], + "comment": "Not supported over CDP" }, { - "testIdPattern": "[browsercontext.spec] BrowserContext should wait for a target", + "testIdPattern": "[ariaqueryhandler.spec] AriaQueryHandler queryOne (Chromium web test) should find by role \"heading\"", "platforms": ["darwin", "linux", "win32"], "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] + "expectations": ["FAIL"], + "comment": "WebDriver BiDi locateNodes does not support shadow roots so far" }, { - "testIdPattern": "[browsercontext.spec] BrowserContext window.open should use parent tab context", + "testIdPattern": "[autofill.spec] *", "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] + "parameters": ["chrome", "chrome-headless-shell"], + "expectations": ["FAIL"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { "testIdPattern": "[CDPSession.spec] Target.createCDPSession *", @@ -313,6 +243,20 @@ "expectations": ["SKIP"], "comment": "The test relies on the default page partition key do not contain the source origin. This is not the case for Firefox." }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set cookie with all available properties", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["FAIL"], + "comment": "Chromium-specific test. The sourceScheme property is chromium-specific." + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set secure same-site cookies from a frame", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["chrome"], + "expectations": ["FAIL"], + "comment": "Failes with fieldtrial testing config in Chrome" + }, { "testIdPattern": "[coverage.spec] *", "platforms": ["darwin", "linux", "win32"], @@ -419,11 +363,11 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[frame.spec] Frame specs Frame Management should report frame.name()", + "testIdPattern": "[frame.spec] Frame specs Frame.prototype.frameElement should handle shadow roots", "platforms": ["darwin", "linux", "win32"], "parameters": ["webDriverBiDi"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "https://github.com/w3c/webdriver-bidi/issues/794" }, { "testIdPattern": "[idle_override.spec] *", @@ -488,27 +432,6 @@ "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.executablePath should work", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should be able to launch Chrome", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should be able to launch Firefox", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should work with no default arguments", "platforms": ["darwin", "linux", "win32"], @@ -523,13 +446,6 @@ "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch userDataDir argument with non-existent dir", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[mouse.spec] Mouse should not throw if buttons are pressed twice", "platforms": ["darwin", "linux", "win32"], @@ -551,6 +467,13 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to URL with hash and fire requests without hash", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "BiDi spec expect the request to not trim the hash" + }, { "testIdPattern": "[navigation.spec] navigation Page.goto should send referer", "platforms": ["darwin", "linux", "win32"], @@ -559,16 +482,25 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[network.spec] network Network Events Page.Events.RequestFinished", + "testIdPattern": "[navigation.spec] navigation Page.goto should work when navigating to a URL with a client redirect", "platforms": ["darwin", "linux", "win32"], "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] + "expectations": ["FAIL"], + "comment": "https://github.com/puppeteer/puppeteer/issues/12929" }, { - "testIdPattern": "[network.spec] network Network Events should fire events in proper order", + "testIdPattern": "[network.spec] network Network Events Page.Events.RequestServedFromCache *", "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["PASS"] + "parameters": ["cdp", "firefox"], + "expectations": ["FAIL"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, + { + "testIdPattern": "[network.spec] network Page.authenticate should not disable caching for *", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["FAIL"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { "testIdPattern": "[network.spec] network Request.initiator should return the initiator", @@ -590,18 +522,10 @@ "expectations": ["PASS"] }, { - "testIdPattern": "[network.spec] network Request.postData should be |undefined| when there is no post data", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "Unsupported" - }, - { - "testIdPattern": "[network.spec] network Request.postData should work", + "testIdPattern": "[network.spec] network Request.isNavigationRequest should work with request interception", "platforms": ["darwin", "linux", "win32"], "parameters": ["webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "expectations": ["PASS"] }, { "testIdPattern": "[network.spec] network Request.postData should work with blobs", @@ -611,30 +535,32 @@ "comment": "Not implemented for BiDi yet." }, { - "testIdPattern": "[network.spec] network Response.fromServiceWorker Response.fromServiceWorker", + "testIdPattern": "[network.spec] network Request.resourceType *", "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "Bidi does not have equivalent field" }, { - "testIdPattern": "[network.spec] network Response.timing returns timing information", + "testIdPattern": "[network.spec] network Response.fromCache should work for *", "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], + "parameters": ["cdp", "firefox"], "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[network.spec] network Page.setBypassServiceWorker *", + "testIdPattern": "[network.spec] network Response.fromCache should work for *", "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["FAIL"] + "parameters": ["chrome", "webDriverBiDi"], + "expectations": ["FAIL", "PASS"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[network.spec] network Page.setBypassServiceWorker *", + "testIdPattern": "[network.spec] network Response.fromServiceWorker Response.fromServiceWorker", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox"], - "expectations": ["FAIL", "TIMEOUT"] + "parameters": ["webDriverBiDi"], + "expectations": ["SKIP"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { "testIdPattern": "[oopif.spec] *", @@ -643,12 +569,19 @@ "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[oopif.spec] OOPIF should evaluate on a page with a PDF viewer", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["chrome-headless-shell"], + "expectations": ["FAIL"], + "comment": "chrome-headless-shell does not have a PDF viewer" + }, { "testIdPattern": "[page.spec] Page Page.addScriptTag should throw when added with content to the CSP page", "platforms": ["darwin", "linux", "win32"], "parameters": ["webDriverBiDi"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "See https://github.com/puppeteer/puppeteer/issues/4840" }, { "testIdPattern": "[page.spec] Page Page.addStyleTag should throw when added with content to the CSP page", @@ -658,16 +591,16 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[page.spec] Page Page.close should *not* run beforeunload by default", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "testIdPattern": "[page.spec] Page Page.bringToFront should work", + "platforms": ["darwin"], + "parameters": ["chrome"], + "expectations": ["FAIL", "PASS"], + "comment": "Flaky on Mac" }, { "testIdPattern": "[page.spec] Page Page.close should *not* run beforeunload by default", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], + "parameters": ["cdp", "firefox"], "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, @@ -707,25 +640,25 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[page.spec] Page Page.exposeFunction *", + "testIdPattern": "[page.spec] Page Page.Events.Popup *", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Deprecated" }, { - "testIdPattern": "[page.spec] Page Page.exposeFunction should work with loading frames", + "testIdPattern": "[page.spec] Page Page.exposeFunction *", "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], + "parameters": ["cdp", "firefox"], "expectations": ["SKIP"], - "comment": "Missing request interception" + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { "testIdPattern": "[page.spec] Page Page.pdf should respect timeout", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox"], "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "https://github.com/puppeteer/puppeteer/issues/12152" }, { "testIdPattern": "[page.spec] Page Page.setBypassCSP *", @@ -735,95 +668,74 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[page.spec] Page Page.setCacheEnabled should stay disabled when toggling request interception on/off", + "testIdPattern": "[pdf.spec] Page.pdf *", "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["FAIL"], + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[page.spec] Page Page.setOfflineMode should emulate navigator.onLine", + "testIdPattern": "[prerender.spec] Prerender can screencast", "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "parameters": ["chrome"], + "expectations": ["SKIP"], + "comment": "Currently no reliable ffmpeg downloads for testing https://github.com/puppeteer/puppeteer/issues/12121" }, { - "testIdPattern": "[page.spec] Page Page.setOfflineMode should work", + "testIdPattern": "[prerender.spec] Prerender can screencast", "platforms": ["darwin", "linux", "win32"], - "parameters": ["webDriverBiDi"], - "expectations": ["FAIL"], + "parameters": ["firefox"], + "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[page.spec] Page Page.setUserAgent *", + "testIdPattern": "[queryObjects.spec] *", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], + "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[pdf.spec] Page.pdf *", + "testIdPattern": "[requestinterception-experimental.spec] *", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[prerender.spec] Prerender can screencast", + "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Page.setRequestInterception should be abortable with custom error codes", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "parameters": ["webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "TODO: BiDi does not support custom errors - https://github.com/w3c/webdriver-bidi/issues/508" }, { - "testIdPattern": "[queryhandler.spec] Query handler tests P selectors should work ARIA selectors", + "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Page.setRequestInterception should navigate to URL with hash and fire requests without hash", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "parameters": ["webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "BiDi spec expect the request to not trim the hash" }, { - "testIdPattern": "[queryhandler.spec] Query handler tests P selectors should work ARIA selectors with name and role", + "testIdPattern": "[requestinterception.spec] *", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox"], + "parameters": ["cdp", "firefox"], "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[queryhandler.spec] Query handler tests P selectors should work ARIA selectors with role", + "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should be abortable with custom error codes", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[queryhandler.spec] Query handler tests P selectors should work for ARIA selectors in multiple isolated worlds", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[queryObjects.spec] *", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[requestinterception-experimental.spec] *", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["FAIL", "SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "parameters": ["webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "https://github.com/w3c/webdriver-bidi/issues/508" }, { - "testIdPattern": "[requestinterception.spec] *", + "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should navigate to URL with hash and fire requests without hash", "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["FAIL", "SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "parameters": ["webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "BiDi spec and WPT require expect the Hash" }, { "testIdPattern": "[screencast.spec] Screencasts Page.screencast should validate options", @@ -884,7 +796,7 @@ "testIdPattern": "[TargetManager.spec] *", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], - "expectations": ["FAIL", "SKIP"], + "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { @@ -909,66 +821,87 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[accessibility.spec] Accessibility get snapshots while the tree is re-calculated", + "testIdPattern": "[acceptInsecureCerts.spec] acceptInsecureCerts Response.securityDetails Network redirects should report SecurityDetails", "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["SKIP"], + "parameters": ["cdp", "firefox"], + "expectations": ["FAIL", "PASS"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[ariaqueryhandler.spec] AriaQueryHandler queryOne (Chromium web test) should find by role \"button\"", + "testIdPattern": "[acceptInsecureCerts.spec] acceptInsecureCerts Response.securityDetails Network redirects should report SecurityDetails", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], - "comment": "Querying by a11y attributes is not standard behavior" + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[ariaqueryhandler.spec] AriaQueryHandler queryOne (Chromium web test) should find by role \"heading\"", + "testIdPattern": "[acceptInsecureCerts.spec] acceptInsecureCerts Response.securityDetails should be |null| for non-secure requests", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], - "comment": "Querying by a11y attributes is not standard behavior" + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[bfcache.spec] BFCache can navigate to a BFCached page containing an OOPIF and a worker", + "testIdPattern": "[acceptInsecureCerts.spec] acceptInsecureCerts Response.securityDetails should work", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], - "expectations": ["SKIP"], + "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[browser.spec] Browser specs Browser.isConnected should set the browser connected state", + "testIdPattern": "[acceptInsecureCerts.spec] acceptInsecureCerts Response.securityDetails should work", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["SKIP"], + "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[browser.spec] Browser specs Browser.process should not return child_process for remote browser", + "testIdPattern": "[acceptInsecureCerts.spec] acceptInsecureCerts should work with request interception", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], + "parameters": ["cdp", "firefox"], + "expectations": ["FAIL"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, + { + "testIdPattern": "[accessibility.spec] Accessibility get snapshots while the tree is re-calculated", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["chrome", "webDriverBiDi"], "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[browser.spec] Browser specs Browser.userAgent should include Browser engine", + "testIdPattern": "[ariaqueryhandler.spec] AriaQueryHandler queryAllArray $$eval should handle many elements", "platforms": ["darwin", "linux", "win32"], "parameters": ["chrome", "webDriverBiDi"], + "expectations": ["PASS", "TIMEOUT"], + "comment": "times out flakily" + }, + { + "testIdPattern": "[bfcache.spec] BFCache can call a function exposed on a page restored from bfcache", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], "expectations": ["FAIL"], + "comment": "addBinding is not supported" + }, + { + "testIdPattern": "[bfcache.spec] BFCache can navigate to a BFCached page containing an OOPIF and a worker", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[browser.spec] Browser specs Browser.userAgent should include Browser engine", + "testIdPattern": "[browser.spec] Browser specs Browser.isConnected should set the browser connected state", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], + "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[browser.spec] Browser specs Browser.version should return version", + "testIdPattern": "[browser.spec] Browser specs Browser.process should not return child_process for remote browser", "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { @@ -978,12 +911,6 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[browsercontext.spec] BrowserContext BrowserContext.overridePermissions should fail when bad permission is given", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["PASS"] - }, { "testIdPattern": "[browsercontext.spec] BrowserContext BrowserContext.overridePermissions should grant permission when listed", "platforms": ["darwin", "linux", "win32"], @@ -1033,6 +960,13 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[browsercontext.spec] BrowserContext should fire target events", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "In BiDi currently more events than needed are fired (because target is updated more often). We probably need to adjust the test as the behavior is not broken per se" + }, { "testIdPattern": "[browsercontext.spec] BrowserContext should wait for a target", "platforms": ["darwin", "linux", "win32"], @@ -1048,32 +982,32 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[CDPSession.spec] Target.createCDPSession should be able to detach session", + "testIdPattern": "[browsercontext.spec] BrowserContext should work across sessions", "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], + "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Firefox does not support multiple connection at this time" }, { - "testIdPattern": "[CDPSession.spec] Target.createCDPSession should enable and disable domains independently", + "testIdPattern": "[CDPSession.spec] Target.createCDPSession should be able to detach session", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[CDPSession.spec] Target.createCDPSession should expose the underlying connection", + "testIdPattern": "[CDPSession.spec] Target.createCDPSession should enable and disable domains independently", "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], + "parameters": ["cdp", "firefox"], "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[CDPSession.spec] Target.createCDPSession should respect custom timeout", + "testIdPattern": "[CDPSession.spec] Target.createCDPSession should not send extra events", "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "parameters": ["cdp", "firefox"], + "expectations": ["FAIL"], + "comment": "Fails with Firefox CDP" }, { "testIdPattern": "[CDPSession.spec] Target.createCDPSession should respect custom timeout", @@ -1103,13 +1037,6 @@ "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[chromiumonly.spec] Chromium-Specific Page Tests Page.setRequestInterception should work with intervention headers", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[click.spec] Page.click should click on checkbox label and toggle", "platforms": ["darwin", "linux", "win32"], @@ -1124,20 +1051,6 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[click.spec] Page.click should click the button with deviceScaleFactor set", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[click.spec] Page.click should click the button with fixed position inside an iframe", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[click.spec] Page.click should click the button with fixed position inside an iframe", "platforms": ["darwin", "linux", "win32"], @@ -1166,6 +1079,13 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[click.spec] Page.click should not throw UnhandledPromiseRejection when page closes", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], + "comment": "The timing has changed and now the target is closed early resulting in Firefox not-resolving Input commands which causes the test to time out" + }, { "testIdPattern": "[click.spec] Page.click should scroll and click with disabled javascript", "platforms": ["darwin", "linux", "win32"], @@ -1209,11 +1129,11 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[cookies.spec] Cookie specs Page.deleteCookie should delete cookie", + "testIdPattern": "[cookies.spec] Cookie specs Page.cookies should not get cookies from subdomain if the cookie is for top-level domain", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], + "parameters": ["cdp", "firefox"], "expectations": ["FAIL"], - "comment": "Firefox default partition key is inconsistent: #12004" + "comment": "Deprecated" }, { "testIdPattern": "[cookies.spec] Cookie specs Page.deleteCookie should delete cookie", @@ -1230,11 +1150,11 @@ "comment": "Not supported with cdp" }, { - "testIdPattern": "[cookies.spec] Cookie specs Page.deleteCookie should not delete cookie for different domain", + "testIdPattern": "[cookies.spec] Cookie specs Page.deleteCookie should delete cookie with partition key if partition key is specified", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], + "parameters": ["cdp", "firefox"], "expectations": ["FAIL"], - "comment": "Firefox default partition key is inconsistent: #12004" + "comment": "Not supported" }, { "testIdPattern": "[cookies.spec] Cookie specs Page.deleteCookie should not delete cookie for different domain", @@ -1243,6 +1163,13 @@ "expectations": ["FAIL"], "comment": "Not supported with cdp" }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.deleteCookie should only delete cookie from the default partition if partitionkey is not specified", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["FAIL"], + "comment": "Not supported" + }, { "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should default to setting secure cookie for HTTPS websites", "platforms": ["darwin", "linux", "win32"], @@ -1265,16 +1192,16 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set a cookie with a path", + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set a cookie with a partitionKey", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Not supported" }, { "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set a cookie with a path", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], + "parameters": ["cdp", "firefox"], "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, @@ -1292,13 +1219,6 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set cookies from a frame", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set multiple cookies", "platforms": ["darwin", "linux", "win32"], @@ -1307,18 +1227,10 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set multiple cookies", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set cookie with all available properties", + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set secure same-site cookies from a frame", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox"], - "expectations": ["FAIL"], - "comment": "Chromium-specific test. The sourceScheme property is chromium-specific." + "parameters": ["chrome", "chrome-headless-shell"], + "expectations": ["PASS"] }, { "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set secure same-site cookies from a frame", @@ -1334,13 +1246,6 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should work", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[debugInfo.spec] DebugInfo Browser.debugInfo should work", "platforms": ["darwin", "linux", "win32"], @@ -1362,13 +1267,6 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[defaultbrowsercontext.spec] DefaultBrowserContext page.deleteCookie() should work", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[defaultbrowsercontext.spec] DefaultBrowserContext page.setCookie() should work", "platforms": ["darwin", "linux", "win32"], @@ -1377,9 +1275,9 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[defaultbrowsercontext.spec] DefaultBrowserContext page.setCookie() should work", + "testIdPattern": "[device-request-prompt.spec] device request prompt can be aborted", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], + "parameters": ["chrome", "webDriverBiDi"], "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, @@ -1426,26 +1324,61 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[emulation.spec] Emulation Page.emulate should support clicking", + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.touchEnd should work", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], - "expectations": ["SKIP"], + "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[emulation.spec] Emulation Page.emulate should support clicking", + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.touchMove should work", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[emulation.spec] Emulation Page.emulate should work", + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.touchMove should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["FAIL"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.touchMove should work with a pre-existing Touch", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.touchMove should work with a pre-existing Touch", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["FAIL"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.touchStart should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["FAIL"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.touchStart should work with the returned Touch", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["FAIL"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulate should support clicking", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, { "testIdPattern": "[emulation.spec] Emulation Page.emulateCPUThrottling should change the CPU throttling rate successfully", "platforms": ["darwin", "linux", "win32"], @@ -1495,13 +1428,6 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[emulation.spec] Emulation Page.emulateNetworkConditions should change navigator.connection.effectiveType", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[emulation.spec] Emulation Page.emulateNetworkConditions should change navigator.connection.effectiveType", "platforms": ["darwin", "linux", "win32"], @@ -1565,13 +1491,6 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[emulation.spec] Emulation Page.viewport should load correct pictures when emulation dpr", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[emulation.spec] Emulation Page.viewport should support landscape emulation", "platforms": ["darwin", "linux", "win32"], @@ -1593,13 +1512,6 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[emulation.spec] Emulation Page.viewport should update media queries when resoltion changes", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should simulate a user gesture", "platforms": ["darwin", "linux", "win32"], @@ -1621,6 +1533,13 @@ "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[extensions.spec] extensions background_page target type should be available", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "chrome"], + "expectations": ["SKIP"], + "comment": "See https://github.com/puppeteer/puppeteer/issues/12866" + }, { "testIdPattern": "[extensions.spec] extensions background_page target type should be available", "platforms": ["darwin", "linux", "win32"], @@ -1638,15 +1557,15 @@ { "testIdPattern": "[extensions.spec] extensions target.page() should return a background_page", "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], + "parameters": ["cdp", "chrome"], "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "See https://github.com/puppeteer/puppeteer/issues/12866" }, { "testIdPattern": "[extensions.spec] extensions target.page() should return a background_page", "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "chrome"], - "expectations": ["FAIL", "PASS"], + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { @@ -1656,6 +1575,13 @@ "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[fixtures.spec] Fixtures should close the browser when the node process closes", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["chrome", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "The browser `disconnect` event is not emitted" + }, { "testIdPattern": "[frame.spec] Frame specs Frame Management should click elements in a frameset", "platforms": ["darwin", "linux", "win32"], @@ -1677,13 +1603,6 @@ "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[frame.spec] Frame specs Frame Management should report frame.name()", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[frame.spec] Frame specs Frame Management should report frame.parent()", "platforms": ["darwin", "linux", "win32"], @@ -1727,81 +1646,17 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[acceptInsecureCerts.spec] ignoreHTTPSErrors Response.securityDetails Network redirects should report SecurityDetails", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[acceptInsecureCerts.spec] ignoreHTTPSErrors Response.securityDetails Network redirects should report SecurityDetails", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["FAIL", "PASS"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[acceptInsecureCerts.spec] ignoreHTTPSErrors Response.securityDetails Network redirects should report SecurityDetails", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[acceptInsecureCerts.spec] ignoreHTTPSErrors Response.securityDetails should be |null| for non-secure requests", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[acceptInsecureCerts.spec] ignoreHTTPSErrors Response.securityDetails should be |null| for non-secure requests", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[acceptInsecureCerts.spec] ignoreHTTPSErrors Response.securityDetails should work", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[acceptInsecureCerts.spec] ignoreHTTPSErrors Response.securityDetails should work", + "testIdPattern": "[frame.spec] Frame specs Frame.prototype.frameElement should handle shadow roots", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[acceptInsecureCerts.spec] ignoreHTTPSErrors Response.securityDetails should work", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "expectations": ["SKIP"], + "comment": "not supported" }, { - "testIdPattern": "[acceptInsecureCerts.spec] ignoreHTTPSErrors should work with request interception", + "testIdPattern": "[idle_override.spec] Emulate idle state changing idle state emulation causes change of the IdleDetector state", "platforms": ["darwin", "linux", "win32"], "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[acceptInsecureCerts.spec] ignoreHTTPSErrors should work with request interception", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[acceptInsecureCerts.spec] ignoreHTTPSErrors should work with request interception", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "expectations": ["PASS"] }, { "testIdPattern": "[input.spec] input tests FileChooser.accept should accept single file", @@ -1971,104 +1826,6 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[input.spec] input tests Page.waitForFileChooser should prioritize exact timeout over default timeout", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[input.spec] input tests Page.waitForFileChooser should prioritize exact timeout over default timeout", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[input.spec] input tests Page.waitForFileChooser should respect default timeout when there is no custom timeout", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[input.spec] input tests Page.waitForFileChooser should respect default timeout when there is no custom timeout", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[input.spec] input tests Page.waitForFileChooser should respect timeout", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[input.spec] input tests Page.waitForFileChooser should respect timeout", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[input.spec] input tests Page.waitForFileChooser should return the same file chooser when there are many watchdogs simultaneously", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[input.spec] input tests Page.waitForFileChooser should return the same file chooser when there are many watchdogs simultaneously", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[input.spec] input tests Page.waitForFileChooser should work when file input is attached to DOM", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[input.spec] input tests Page.waitForFileChooser should work when file input is attached to DOM", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[input.spec] input tests Page.waitForFileChooser should work when file input is not attached to DOM", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[input.spec] input tests Page.waitForFileChooser should work when file input is not attached to DOM", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[input.spec] input tests Page.waitForFileChooser should work with no timeout", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[input.spec] input tests Page.waitForFileChooser should work with no timeout", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[keyboard.spec] Keyboard ElementHandle.press should not support |text| option", "platforms": ["darwin", "linux", "win32"], @@ -2181,13 +1938,6 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Browser.disconnect should reject navigation when browser closes", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["SKIP"], - "comment": "https://github.com/puppeteer/puppeteer/issues/11849" - }, { "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Browser.disconnect should reject navigation when browser closes", "platforms": ["darwin", "linux", "win32"], @@ -2200,7 +1950,7 @@ "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Requires support for multiple sessions" }, { "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should be able to close remote browser", @@ -2244,20 +1994,6 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should be able to reconnect", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should be able to reconnect to a disconnected browser", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should be able to reconnect to a disconnected browser", "platforms": ["darwin", "linux", "win32"], @@ -2273,18 +2009,18 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should support ignoreHTTPSErrors option", + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should support acceptInsecureCerts option", "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], + "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Firefox does not support multiple sessions in BiDi." }, { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should support ignoreHTTPSErrors option", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should support acceptInsecureCerts option", + "platforms": ["darwin"], + "parameters": ["chrome", "headless"], + "expectations": ["SKIP"], + "comment": "Started to flake on macos-latest (arm64)" }, { "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should support targetFilter option", @@ -2322,25 +2058,11 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should be able to launch Chrome", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should be able to launch Firefox", + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch can launch multiple instances without node warnings", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should be able to launch Firefox", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "headless"], - "expectations": ["FAIL", "PASS"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], + "comment": "https://bugzilla.mozilla.org/show_bug.cgi?id=1908031" }, { "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should close browser with beforeunload page", @@ -2359,9 +2081,9 @@ { "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should filter out ignored default arguments in Chrome", "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], + "parameters": ["firefox", "headful"], "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Test is testing chrome specific ignore and fails" }, { "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should have custom URL when launching browser", @@ -2378,18 +2100,11 @@ "comment": "For some reason it times out on MacOS. Perhaps it's a bug in chromium-bidi" }, { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch tmp profile should be cleaned up", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch userDataDir option should restore cookies", + "platforms": ["darwin"], + "parameters": ["cdp", "chrome-headless-shell"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch tmp profile should be cleaned up", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Started to fail on Mac with Chrome 130. TODO: file an issue." }, { "testIdPattern": "[locator.spec] Locator Locator.race races multiple locators", @@ -2405,6 +2120,13 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[mouse.spec] Mouse should set ctrlKey on the wheel event", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], + "comment": "Wheel event is not supported" + }, { "testIdPattern": "[mouse.spec] Mouse should trigger hover state with removed window.Node", "platforms": ["darwin", "linux", "win32"], @@ -2454,6 +2176,13 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[navigation.spec] navigation Page.goBack should work with HistoryAPI", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "https://github.com/w3c/webdriver-bidi/issues/502" + }, { "testIdPattern": "[navigation.spec] navigation Page.goto should fail when navigating to bad SSL", "platforms": ["darwin", "linux", "win32"], @@ -2475,13 +2204,6 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to dataURL and fire dataURL requests", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to empty page with networkidle0", "platforms": ["darwin", "linux", "win32"], @@ -2511,18 +2233,11 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to URL with hash and fire requests without hash", + "testIdPattern": "[navigation.spec] navigation Page.goto should return response when page replaces its state during load", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[navigation.spec] navigation Page.goto should not leak listeners during navigation", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], + "parameters": ["cdp", "firefox"], "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "times out with Firefox CDP" }, { "testIdPattern": "[navigation.spec] navigation Page.goto should send referer", @@ -2538,13 +2253,6 @@ "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[navigation.spec] navigation Page.goto should work when navigating to data url", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[navigation.spec] navigation Page.goto should work when navigating to data url", "platforms": ["darwin", "linux", "win32"], @@ -2552,13 +2260,6 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[navigation.spec] navigation Page.goto should work when page calls history API in beforeunload", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "History navigation is breaking the Puppeteer expecation about navigation." - }, { "testIdPattern": "[navigation.spec] navigation Page.goto should work with anchor navigation", "platforms": ["darwin", "linux", "win32"], @@ -2581,39 +2282,39 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work when subframe issues window.stop()", + "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with DOM history.back()/history.forward()", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["SKIP"], + "parameters": ["cdp", "firefox"], + "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work when subframe issues window.stop()", + "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with DOM history.back()/history.forward()", "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], + "parameters": ["firefox", "webDriverBiDi"], "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Test timeout: https://github.com/w3c/webdriver-bidi/issues/502" }, { - "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with DOM history.back()/history.forward()", + "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with history.pushState()", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], - "expectations": ["FAIL"], + "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with history.pushState()", "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], + "parameters": ["firefox", "webDriverBiDi"], "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Test timeout: https://github.com/w3c/webdriver-bidi/issues/502" }, { "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with history.replaceState()", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], "expectations": ["SKIP"], - "comment": "https://github.com/puppeteer/puppeteer/issues/11854" + "comment": "Test timeout: https://github.com/w3c/webdriver-bidi/issues/502" }, { "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with history.replaceState()", @@ -2641,7 +2342,7 @@ "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Fetch.enable not supported with CDP" }, { "testIdPattern": "[network.spec] network Network Events Page.Events.RequestFinished", @@ -2651,17 +2352,11 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[network.spec] network Network Events Page.Events.RequestServedFromCache", + "testIdPattern": "[network.spec] network Network Events Page.Events.RequestServedFromCache for stylesheet", "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], + "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[network.spec] network Network Events Page.Events.RequestServedFromCache", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["PASS"] + "comment": "Needs Firefox support for network events for cached stylesheets https://bugzilla.mozilla.org/show_bug.cgi?id=1879438" }, { "testIdPattern": "[network.spec] network Network Events Page.Events.Response", @@ -2706,52 +2401,73 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[network.spec] network Page.authenticate should fail if wrong credentials", + "testIdPattern": "[network.spec] network Page.authenticate should allow disable authentication", "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["SKIP"], + "comment": "The Puppeteer implementation does not expect 2 responseCompleted events (that AuthRequired triggered); times out sometimes." }, { - "testIdPattern": "[network.spec] network Page.authenticate should not disable caching", + "testIdPattern": "[network.spec] network Page.authenticate should error if authentication is required but not enabled", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "expectations": ["SKIP"], + "comment": "Not supported" }, { - "testIdPattern": "[network.spec] network Page.authenticate should work", + "testIdPattern": "[network.spec] network Page.authenticate should error if authentication is required but not enabled", "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], + "parameters": ["chrome", "webDriverBiDi"], + "expectations": ["FAIL", "PASS"], + "comment": "Flaky see https://github.com/puppeteer/puppeteer/issues/12253" + }, + { + "testIdPattern": "[network.spec] network Page.authenticate should error if authentication is required but not enabled", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "When navigating to a page with authentication the navigate command does not error and never resolves due to the auth dialog being shown https://github.com/w3c/webdriver-bidi/issues/719" }, { - "testIdPattern": "[network.spec] network Page.setExtraHTTPHeaders should work", + "testIdPattern": "[network.spec] network Page.authenticate should fail if wrong credentials", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[network.spec] network raw network headers Cross-origin set-cookie", + "testIdPattern": "[network.spec] network Page.authenticate should not disable caching for script", "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL", "PASS"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["SKIP"], + "comment": "TODO: Needs investigation, isCached is false for the script request handle via network interception" }, { - "testIdPattern": "[network.spec] network raw network headers Same-origin set-cookie navigation", + "testIdPattern": "[network.spec] network Page.authenticate should not disable caching for stylesheet", "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["SKIP"], + "comment": "Needs Firefox support for network events for cached stylesheets https://bugzilla.mozilla.org/show_bug.cgi?id=1879438" + }, + { + "testIdPattern": "[network.spec] network Page.authenticate should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], + "comment": "Not supported" + }, + { + "testIdPattern": "[network.spec] network Page.setExtraHTTPHeaders should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[network.spec] network raw network headers Same-origin set-cookie subresource", + "testIdPattern": "[network.spec] network raw network headers Cross-origin set-cookie", "platforms": ["darwin", "linux", "win32"], "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], + "expectations": ["FAIL", "PASS"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { @@ -2789,6 +2505,20 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[network.spec] network Request.postData should be |undefined| when there is no post data", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "Unsupported" + }, + { + "testIdPattern": "[network.spec] network Request.postData should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, { "testIdPattern": "[network.spec] network Request.postData should work", "platforms": ["darwin", "linux", "win32"], @@ -2825,38 +2555,31 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[network.spec] network Response.fromCache should work", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[network.spec] network Response.fromCache should work", + "testIdPattern": "[network.spec] network Response.fromCache should work for stylesheet", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], - "comment": "Needs investigation, it looks like a bug in Puppeteer" + "comment": "Needs Firefox support for network events for cached stylesheets https://bugzilla.mozilla.org/show_bug.cgi?id=1879438" }, { - "testIdPattern": "[network.spec] network Response.fromCache should work", + "testIdPattern": "[network.spec] network Response.fromServiceWorker Response.fromServiceWorker", "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL", "PASS"], + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[network.spec] network Response.fromServiceWorker Response.fromServiceWorker", + "testIdPattern": "[network.spec] network Response.json should work", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[network.spec] network Response.json should work", + "testIdPattern": "[network.spec] network Response.remoteAddress should support redirects", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], - "expectations": ["SKIP"], + "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { @@ -2902,60 +2625,32 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[oopif.spec] OOPIF should expose events within OOPIFs", + "testIdPattern": "[oopif.spec] OOPIF should evaluate on a page with a PDF viewer", "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], + "parameters": ["firefox", "webDriverBiDi"], "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "chrome-specific test" }, { "testIdPattern": "[oopif.spec] OOPIF should expose events within OOPIFs", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[oopif.spec] OOPIF should keep track of a frames OOP state", - "platforms": ["darwin", "linux", "win32"], "parameters": ["chrome", "webDriverBiDi"], "expectations": ["SKIP"], - "comment": "Failed previously and currently times out" - }, - { - "testIdPattern": "[oopif.spec] OOPIF should load oopif iframes with subresources and request interception", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[oopif.spec] OOPIF should load oopif iframes with subresources and request interception", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[oopif.spec] OOPIF should report google.com frame", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[oopif.spec] OOPIF should report google.com frame", + "testIdPattern": "[oopif.spec] OOPIF should expose events within OOPIFs", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[oopif.spec] OOPIF should recover cross-origin frames on reconnect", + "testIdPattern": "[oopif.spec] OOPIF should keep track of a frames OOP state", "platforms": ["darwin", "linux", "win32"], "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "See https://github.com/GoogleChromeLabs/chromium-bidi/issues/2362" + "expectations": ["SKIP"], + "comment": "Failed previously and currently times out" }, { "testIdPattern": "[oopif.spec] OOPIF should recover cross-origin frames on reconnect", @@ -2968,8 +2663,8 @@ "testIdPattern": "[oopif.spec] OOPIF should support lazy OOP frames", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "https://bugzilla.mozilla.org/show_bug.cgi?id=187816" + "expectations": ["FAIL", "TIMEOUT"], + "comment": "https://bugzilla.mozilla.org/show_bug.cgi?id=1878166" }, { "testIdPattern": "[oopif.spec] OOPIF should wait for inner OOPIFs", @@ -2983,7 +2678,7 @@ "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "chrome"], "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "See https://github.com/puppeteer/puppeteer/issues/4840" }, { "testIdPattern": "[page.spec] Page Page.addScriptTag should throw when added with content to the CSP page", @@ -2993,59 +2688,31 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[page.spec] Page Page.bringToFront should work", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[page.spec] Page Page.bringToFront should work", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[page.spec] Page Page.close should reject all promises when page is closed", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[page.spec] Page Page.close should run beforeunload if asked for", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[page.spec] Page Page.close should run beforeunload if asked for", + "testIdPattern": "[page.spec] Page Page.bringToFront should work", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[page.spec] Page Page.close should terminate network waiters", + "testIdPattern": "[page.spec] Page Page.close should reject all promises when page is closed", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[page.spec] Page Page.Events.Console should have location and stack trace for console API calls", + "testIdPattern": "[page.spec] Page Page.close should run beforeunload if asked for", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], - "expectations": ["FAIL"], + "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[page.spec] Page Page.Events.Console should have location and stack trace for console API calls", + "testIdPattern": "[page.spec] Page Page.close should terminate network waiters", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { @@ -3077,11 +2744,11 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[page.spec] Page Page.Events.Console should work", + "testIdPattern": "[page.spec] Page Page.Events.Console should work for different console API calls with group functions", "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], + "parameters": ["cdp", "firefox"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "We don't normalize the values for CDP as it's being deprecated" }, { "testIdPattern": "[page.spec] Page Page.Events.Console should work for different console API calls with logging functions", @@ -3095,7 +2762,7 @@ "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "https://bugzilla.mozilla.org/show_bug.cgi?id=1866749" }, { "testIdPattern": "[page.spec] Page Page.Events.error should throw when page crashes", @@ -3104,48 +2771,6 @@ "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[page.spec] Page Page.Events.Popup should work", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and rel=noopener", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and with rel=opener", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and without rel=opener", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[page.spec] Page Page.Events.Popup should work with fake-clicking target=_blank and rel=noopener", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[page.spec] Page Page.Events.Popup should work with noopener", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[page.spec] Page Page.metrics metrics event fired on console.timeStamp", "platforms": ["darwin", "linux", "win32"], @@ -3209,13 +2834,6 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[page.spec] Page Page.setCacheEnabled should enable or disable the cache based on the state passed", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[page.spec] Page Page.setCacheEnabled should stay disabled when toggling request interception on/off", "platforms": ["darwin", "linux", "win32"], @@ -3251,6 +2869,13 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[page.spec] Page Page.setOfflineMode should emulate navigator.onLine", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "Not supported by WebDriver BiDi" + }, { "testIdPattern": "[page.spec] Page Page.setOfflineMode should emulate navigator.onLine", "platforms": ["darwin", "linux", "win32"], @@ -3261,30 +2886,30 @@ { "testIdPattern": "[page.spec] Page Page.setOfflineMode should work", "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], + "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Not supported by WebDriver BiDi" }, { - "testIdPattern": "[page.spec] Page Page.setUserAgent should work with additional userAgentMetdata", + "testIdPattern": "[page.spec] Page Page.setOfflineMode should work", "platforms": ["darwin", "linux", "win32"], "parameters": ["cdp", "firefox"], "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[pdf.spec] Page.pdf can print to PDF with outline", + "testIdPattern": "[page.spec] Page Page.setUserAgent should work with additional userAgentMetdata", "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "headful"], + "parameters": ["cdp", "firefox"], "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[pdf.spec] Page.pdf can print to PDF with outline", + "testIdPattern": "[page.spec] Page Page.setUserAgent should work with additional userAgentMetdata", "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "headless"], + "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Chrome-specific test" }, { "testIdPattern": "[proxy.spec] request proxy in incognito browser context should proxy requests when configured at context level", @@ -3307,6 +2932,34 @@ "expectations": ["FAIL", "PASS"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[queryhandler.spec] Query handler tests P selectors should work ARIA selectors", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, + { + "testIdPattern": "[queryhandler.spec] Query handler tests P selectors should work ARIA selectors with name and role", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, + { + "testIdPattern": "[queryhandler.spec] Query handler tests P selectors should work ARIA selectors with role", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, + { + "testIdPattern": "[queryhandler.spec] Query handler tests P selectors should work for ARIA selectors in multiple isolated worlds", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["SKIP"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, { "testIdPattern": "[queryObjects.spec] page.queryObjects should fail for disposed handles", "platforms": ["darwin", "linux", "win32"], @@ -3334,17 +2987,52 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[requestinterception-experimental.spec] request interception \"after each\" hook in \"request interception\"", - "platforms": ["win32"], - "parameters": ["cdp", "chrome"], - "expectations": ["FAIL", "PASS"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Page.setRequestInterception should cache stylesheet if cache enabled", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "Needs Firefox support for network events for cached stylesheets https://bugzilla.mozilla.org/show_bug.cgi?id=1879438" }, { - "testIdPattern": "[requestinterception-experimental.spec] request interception Page.setRequestInterception should load fonts if cache enabled", + "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Page.setRequestInterception should work with file URLs", "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "chrome"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["SKIP"], + "comment": "TODO: Needs support for file URIs in Firefox https://bugzilla.mozilla.org/show_bug.cgi?id=1826210" + }, + { + "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Page.setRequestInterception should work with requests without networkId", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], "expectations": ["SKIP"], + "comment": "Test requires CDP" + }, + { + "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Request.continue should redirect in a way non-observable to page", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "Firefox needs support for the url parameter of continueRequest" + }, + { + "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Request.resourceType should work for document type", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "`HTTPRequest.resourceType()` has no eqivalent in BiDi spec" + }, + { + "testIdPattern": "[requestinterception-experimental.spec] cooperative request interception Request.resourceType should work for stylesheets", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "`HTTPRequest.resourceType()` has no eqivalent in BiDi spec" + }, + { + "testIdPattern": "[requestinterception-experimental.spec] request interception \"after each\" hook in \"request interception\"", + "platforms": ["win32"], + "parameters": ["cdp", "chrome"], + "expectations": ["FAIL", "PASS"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { @@ -3361,6 +3049,20 @@ "expectations": ["FAIL", "PASS"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should be abortable with custom error codes", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["SKIP"], + "comment": "TODO: BiDi does not support custom errors - https://github.com/w3c/webdriver-bidi/issues/508" + }, + { + "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should cache stylesheet if cache enabled", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "Needs Firefox support for network events for cached stylesheets https://bugzilla.mozilla.org/show_bug.cgi?id=1879438" + }, { "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should navigate to URL with hash and fire requests without hash", "platforms": ["darwin", "linux", "win32"], @@ -3369,38 +3071,52 @@ "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[screenshot.spec] Screenshots Cdp should use scale for clip", + "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should work with file URLs", "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], - "expectations": ["FAIL", "PASS"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["SKIP"], + "comment": "TODO: Needs support for file URIs in Firefox https://bugzilla.mozilla.org/show_bug.cgi?id=1826210" }, { - "testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should work for an element with an offset", + "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should work with requests without networkId", "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["SKIP"], + "comment": "Test requires CDP" + }, + { + "testIdPattern": "[requestinterception.spec] request interception Page.setRequestInterception should work with requests without networkId", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["chrome", "webDriverBiDi"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "CDP specific issue, maybe we can support it from BiDi+" }, { - "testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should work for an element with an offset", + "testIdPattern": "[requestinterception.spec] request interception Request.continue should redirect in a way non-observable to page", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "Firefox needs support for the url parameter of continueRequest" }, { - "testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should work with a rotated element", + "testIdPattern": "[requestinterception.spec] request interception Request.resourceType should work for document type", "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "firefox"], + "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + "comment": "`HTTPRequest.resourceType()` has no eqivalent in BiDi spec" }, { - "testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should work with a rotated element", + "testIdPattern": "[requestinterception.spec] request interception Request.resourceType should work for stylesheets", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], + "comment": "`HTTPRequest.resourceType()` has no eqivalent in BiDi spec" + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Cdp should use scale for clip", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["FAIL", "PASS"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { @@ -3410,6 +3126,13 @@ "expectations": ["FAIL", "PASS"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should restore to original viewport size after taking fullPage screenshots when defaultViewport is null", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["cdp", "firefox"], + "expectations": ["FAIL"], + "comment": "Firefox does not support Emulation.clearDeviceMetricsOverride" + }, { "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should take fullPage screenshots", "platforms": ["darwin", "linux", "win32"], @@ -3573,12 +3296,12 @@ { "testIdPattern": "[touchscreen.spec] Touchscreen Touchscreen.prototype.tap should work", "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], + "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[touchscreen.spec] Touchscreen Touchscreen.prototype.tap should work", + "testIdPattern": "[touchscreen.spec] Touchscreen Touchscreen.prototype.tap should work if another touch is already active", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], @@ -3587,12 +3310,26 @@ { "testIdPattern": "[touchscreen.spec] Touchscreen Touchscreen.prototype.touchMove should work", "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], + "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, { - "testIdPattern": "[touchscreen.spec] Touchscreen Touchscreen.prototype.touchMove should work", + "testIdPattern": "[touchscreen.spec] Touchscreen Touchscreen.prototype.touchMove should work when moving touches separately", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, + { + "testIdPattern": "[touchscreen.spec] Touchscreen Touchscreen.prototype.touchMove should work with three touches", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, + { + "testIdPattern": "[touchscreen.spec] Touchscreen Touchscreen.prototype.touchMove should work with two touches", "platforms": ["darwin", "linux", "win32"], "parameters": ["firefox", "webDriverBiDi"], "expectations": ["FAIL"], @@ -3640,20 +3377,6 @@ "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[waittask.spec] waittask specs Frame.waitForSelector should survive cross-process navigation", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["chrome", "webDriverBiDi"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, - { - "testIdPattern": "[waittask.spec] waittask specs Frame.waitForSelector should survive cross-process navigation", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["firefox", "webDriverBiDi"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[waittask.spec] waittask specs Frame.waitForSelector should survive cross-process navigation", "platforms": ["darwin", "linux", "win32"], @@ -3696,6 +3419,12 @@ "expectations": ["FAIL"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, + { + "testIdPattern": "[worker.spec] Workers should report errors", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "webDriverBiDi"], + "expectations": ["PASS"] + }, { "testIdPattern": "[CDPSession.spec] Target.createCDPSession should send events", "platforms": ["win32"], @@ -3731,13 +3460,6 @@ "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[extensions.spec] extensions background_page target type should be available", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "chrome", "chrome-headless-shell"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[extensions.spec] extensions service_worker target type should be available", "platforms": ["darwin", "linux", "win32"], @@ -3745,13 +3467,6 @@ "expectations": ["SKIP"], "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" }, - { - "testIdPattern": "[extensions.spec] extensions target.page() should return a background_page", - "platforms": ["darwin", "linux", "win32"], - "parameters": ["cdp", "chrome", "chrome-headless-shell"], - "expectations": ["SKIP"], - "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" - }, { "testIdPattern": "[network.spec] network Network Events Page.Events.Request", "platforms": ["linux"], @@ -3801,6 +3516,33 @@ "expectations": ["SKIP"], "comment": "For some reason service_workers do not close in chrome-headless-shell" }, + { + "testIdPattern": "[touchscreen.spec] Touchscreen Touchscreen.prototype.touchMove should throw if no touch was started", + "platforms": ["linux"], + "parameters": ["cdp", "firefox", "headless"], + "expectations": ["PASS"] + }, + { + "testIdPattern": "[touchscreen.spec] Touchscreen Touchscreen.prototype.touchMove should work with three touches", + "platforms": ["darwin"], + "parameters": ["cdp", "chrome", "headless"], + "expectations": ["FAIL"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, + { + "testIdPattern": "[touchscreen.spec] Touchscreen Touchscreen.prototype.touchMove should work with three touches", + "platforms": ["darwin"], + "parameters": ["cdp", "chrome", "chrome-headless-shell"], + "expectations": ["FAIL"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, + { + "testIdPattern": "[touchscreen.spec] Touchscreen Touchscreen.prototype.touchMove should work with three touches", + "platforms": ["darwin"], + "parameters": ["chrome", "headless", "webDriverBiDi"], + "expectations": ["FAIL"], + "comment": "TODO: add a comment explaining why this expectation is required (include links to issues)" + }, { "testIdPattern": "[worker.spec] Workers Page.workers", "platforms": ["darwin", "linux", "win32"],