Add Google-specific user-agent tokens and strings

admariner · May 20, 2024 · 60cef1f · 60cef1f
1 parent 577dd3f
commit 60cef1f
Show file tree

Hide file tree

Showing 5 changed files with 81 additions and 1 deletion.
diff --git a/advertools/code_recipes/spider_strategies.py b/advertools/code_recipes/spider_strategies.py
@@ -453,6 +453,26 @@
 Xbox One S                                                Mozilla/5.0 (Windows NT 10.0; Win64; x64; XBOX_ONE_ED) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393
 Xbox Series X                                             Mozilla/5.0 (Windows NT 10.0; Win64; x64; Xbox; Xbox Series X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.82 Safari/537.36 Edge/20.02
 Yahoo! bot                                                Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)
+Googlebot Smartphone                                      Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+Googlebot Desktop                                         Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/W.X.Y.Z Safari/537.36
+Googlebot-Image                                           Googlebot-Image/1.0
+Googlebot-News                                            Googlebot-News
+Googlebot-Video                                           Googlebot-Video/1.0
+Storebot-Google Desktop                                   Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36
+Storebot-Google Smartphone                                Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36
+Google-InspectionTool Mobile                              Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-InspectionTool/1.0;)
+Google-InspectionTool Desktop                             Mozilla/5.0 (compatible; Google-InspectionTool/1.0;)
+GoogleOther                                               Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; GoogleOther)
+GoogleOther-Image                                         GoogleOther-Image/1.0
+GoogleOther-Video                                         GoogleOther-Video/1.0
+APIs-Google                                               APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)
+AdsBot-Google-Mobile                                      Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)
+AdsBot-Google                                             AdsBot-Google (+http://www.google.com/adsbot.html)
+Mediapartners-Google                                      Mediapartners-Google
+Google-Safety                                             Google-Safety
+FeedFetcher-Google                                        FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)
+Google Publisher Center                                   GoogleProducer; (+http://goo.gl/7y4SX)
+Google Site Verifier                                      Mozilla/5.0 (compatible; Google-Site-Verification/1.0)
 ========================================================  =========================================================================================================================================================================
 
 """  # noqa: E501
diff --git a/docs/_build/doctrees/advertools.code_recipes.spider_strategies.doctree b/docs/_build/doctrees/advertools.code_recipes.spider_strategies.doctree
diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle
diff --git a/docs/_build/html/advertools.code_recipes.spider_strategies.html b/docs/_build/html/advertools.code_recipes.spider_strategies.html
@@ -887,6 +887,66 @@ <h2>User-agent strings for use in crawling<a class="headerlink" href="#user-agen
 <tr class="row-odd"><td><p>Yahoo! bot</p></td>
 <td><p>Mozilla/5.0 (compatible; Yahoo! Slurp; <a class="reference external" href="http://help.yahoo.com/help/us/ysearch/slurp">http://help.yahoo.com/help/us/ysearch/slurp</a>)</p></td>
 </tr>
+<tr class="row-even"><td><p>Googlebot Smartphone</p></td>
+<td><p>Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)</p></td>
+</tr>
+<tr class="row-odd"><td><p>Googlebot Desktop</p></td>
+<td><p>Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/W.X.Y.Z Safari/537.36</p></td>
+</tr>
+<tr class="row-even"><td><p>Googlebot-Image</p></td>
+<td><p>Googlebot-Image/1.0</p></td>
+</tr>
+<tr class="row-odd"><td><p>Googlebot-News</p></td>
+<td><p>Googlebot-News</p></td>
+</tr>
+<tr class="row-even"><td><p>Googlebot-Video</p></td>
+<td><p>Googlebot-Video/1.0</p></td>
+</tr>
+<tr class="row-odd"><td><p>Storebot-Google Desktop</p></td>
+<td><p>Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Safari/537.36</p></td>
+</tr>
+<tr class="row-even"><td><p>Storebot-Google Smartphone</p></td>
+<td><p>Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36</p></td>
+</tr>
+<tr class="row-odd"><td><p>Google-InspectionTool Mobile</p></td>
+<td><p>Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Google-InspectionTool/1.0;)</p></td>
+</tr>
+<tr class="row-even"><td><p>Google-InspectionTool Desktop</p></td>
+<td><p>Mozilla/5.0 (compatible; Google-InspectionTool/1.0;)</p></td>
+</tr>
+<tr class="row-odd"><td><p>GoogleOther</p></td>
+<td><p>Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; GoogleOther)</p></td>
+</tr>
+<tr class="row-even"><td><p>GoogleOther-Image</p></td>
+<td><p>GoogleOther-Image/1.0</p></td>
+</tr>
+<tr class="row-odd"><td><p>GoogleOther-Video</p></td>
+<td><p>GoogleOther-Video/1.0</p></td>
+</tr>
+<tr class="row-even"><td><p>APIs-Google</p></td>
+<td><p>APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)</p></td>
+</tr>
+<tr class="row-odd"><td><p>AdsBot-Google-Mobile</p></td>
+<td><p>Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)</p></td>
+</tr>
+<tr class="row-even"><td><p>AdsBot-Google</p></td>
+<td><p>AdsBot-Google (+http://www.google.com/adsbot.html)</p></td>
+</tr>
+<tr class="row-odd"><td><p>Mediapartners-Google</p></td>
+<td><p>Mediapartners-Google</p></td>
+</tr>
+<tr class="row-even"><td><p>Google-Safety</p></td>
+<td><p>Google-Safety</p></td>
+</tr>
+<tr class="row-odd"><td><p>FeedFetcher-Google</p></td>
+<td><p>FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)</p></td>
+</tr>
+<tr class="row-even"><td><p>Google Publisher Center</p></td>
+<td><p>GoogleProducer; (+http://goo.gl/7y4SX)</p></td>
+</tr>
+<tr class="row-odd"><td><p>Google Site Verifier</p></td>
+<td><p>Mozilla/5.0 (compatible; Google-Site-Verification/1.0)</p></td>
+</tr>
 </tbody>
 </table>
 </section>

diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js