tests

Author: Nikolai Tschacher

GoogleScraper/parsing.py

@@ -477,19 +477,19 @@ class YandexParser(Parser):
     # @TODO: In december 2015, I saw that yandex only shows the number of search results in the search input field
     # with javascript. One can scrape it in plain http mode, but the values are hidden in some javascript and not
     # accessible with normal xpath/css selectors. A normal text search is done.
-    num_results_search_selectors = ['.serp-adv .serp-item__wrap > strong', '.input__found_visibility_visible font font::text']
+    num_results_search_selectors = ['.serp-list .serp-adv__found::text', '.input__found_visibility_visible font font::text']
 
     page_number_selectors = ['.pager__group .button_checked_yes span::text']
 
     normal_search_selectors = {
         'results': {
             'de_ip': {
-                'container': 'div.serp-list',
-                'result_container': 'div.serp-item',
-                'link': 'a.serp-item__title-link::attr(href)',
-                'snippet': 'div.serp-item__text::text',
-                'title': 'a.serp-item__title-link::text',
-                'visible_link': 'a.serp-url__link::attr(href)'
+                'container': '.serp-list',
+                'result_container': '.serp-item',
+                'link': 'a.link::attr(href)',
+                'snippet': 'div.text-container::text',
+                'title': 'div.organic__url-text::text',
+                'visible_link': '.typo_type_greenurl::text'
             }
         }
     }

GoogleScraper/scrape_config.py

@@ -99,17 +99,29 @@
 # The google base search url
 google_search_url = 'https://www.google.com/search?'
 
+# whether to change the search settings prior to scraping
+# when this is set to False google will search with 
+# the default search settings that your browser supports
+google_selenium_search_settings = False
+
+# the following options only take effect when 
+# google_selenium_search_settings is set to True
+
 # Search Settings for Google Scraping in Selenium Mode
 # 10, 20, 30, 50, 100
-google_selenium_num_results = 100 
+google_selenium_num_results = 100
 # Private results help find more relevant content for you, including content and connections that only you can see.
 google_selenium_personalization = False
 # use a country code such as US, DE, GB, CH, ...
-google_selenium_region = 'DE' 
+google_selenium_region = 'DE'
 google_selenium_safe_search = False
 # the language for google search results
 google_selenium_language = 'English'
 
+# manually select search settings
+# only possible in visible browsers
+google_selenium_manual_settings = False
+
 
 # The yandex base search url
 yandex_search_url = 'http://yandex.ru/yandsearch?'

GoogleScraper/selenium_mode.py

@@ -59,7 +59,7 @@ class SelScrape(SearchEngineScrape, threading.Thread):
 
     next_page_selectors = {
         'google': '#pnnext',
-        'yandex': '.pager__button_kind_next',
+        'yandex': '.pager__item_kind_next',
         'bing': '.sb_pagN',
         'yahoo': '#pg-next',
         'baidu': '.n',
@@ -474,7 +474,7 @@ def wait_until_serp_loaded(self):
             if self.search_engine_name == 'google':
                 selector = '#navcnt td.cur'
             elif self.search_engine_name == 'yandex':
-                selector = '.pager__item_current_yes font font'
+                selector = '.pager__item_current_yes'
             elif self.search_engine_name == 'bing':
                 selector = 'nav li a.sb_pagS'
             elif self.search_engine_name == 'yahoo':
@@ -497,7 +497,7 @@ def wait_until_serp_loaded(self):
                 except TimeoutException as e:
                     self._save_debug_screenshot()
                     content = self.webdriver.find_element_by_css_selector(selector).text
-                    raise Exception('Pagenumber={} did not appear in navigation. Got "{}" instead'.format(self.page_number, content))
+                    logger.error('Pagenumber={} did not appear in navigation. Got "{}" instead'.format(self.page_number, content))
 
         elif self.search_type == 'image':
             self.wait_until_title_contains_keyword()
@@ -700,63 +700,68 @@ def build_search(self):
         This is highly sensitive.
         """
         super().build_search()
-        # assume we are on the normal google search page right now
-        self.webdriver.get('https://www.google.com/preferences?hl=en')
 
-        time.sleep(1)
+        if self.config.get('google_selenium_search_settings', False):
+            # assume we are on the normal google search page right now
+            self.webdriver.get('https://www.google.com/preferences?hl=en')
 
-        # wait until we see the settings
-        element = WebDriverWait(self.webdriver, 7).until(EC.presence_of_element_located((By.NAME, 'safeui')))
+            time.sleep(1)
 
-        try:
-            if self.config.get('google_selenium_safe_search', False):
-                if self.webdriver.find_element_by_name('safeui').get_attribute('value') != 'on':
-                    self.webdriver.find_element_by_name('safeui').click()
+            if self.config.get('google_selenium_manual_settings', False):
+                return input('Press any Key after search settings completed...')
+
+            # wait until we see the settings
+            element = WebDriverWait(self.webdriver, 7).until(EC.presence_of_element_located((By.NAME, 'safeui')))
 
             try:
-                if self.config.get('google_selenium_personalization', False):
-                    self.webdriver.find_element_by_css_selector('#pson-radio > div:first-child').click()
-                else:
-                    self.webdriver.find_element_by_css_selector('#pson-radio > div:nth-child(2)').click()
-            except WebDriverException as e:
-                logger.warning('Cannot set personalization settings.')
+                if self.config.get('google_selenium_safe_search', False):
+                    if self.webdriver.find_element_by_name('safeui').get_attribute('value') != 'on':
+                        self.webdriver.find_element_by_name('safeui').click()
 
-            time.sleep(1)
+                try:
+                    if self.config.get('google_selenium_personalization', False):
+                        self.webdriver.find_element_by_css_selector('#pson-radio > div:first-child').click()
+                    else:
+                        self.webdriver.find_element_by_css_selector('#pson-radio > div:nth-child(2)').click()
+                except WebDriverException as e:
+                    logger.warning('Cannot set personalization settings.')
 
-            # set the region
-            try:
-                self.webdriver.find_element_by_id('regionanchormore').click()
-            except WebDriverException as e:
-                logger.warning('Regions probably already expanded.')
-
-            region = self.config.get('google_selenium_region', 'US')
-            self.webdriver.find_element_by_css_selector('div[data-value="{}"]'.format(region)).click()
-
-            # set the number of results
-            num_results = self.config.get('google_selenium_num_results', 10)
-            self.webdriver.find_element_by_id('result_slider').click()
-            # reset
-            for i in range(5):
-                self.webdriver.find_element_by_id('result_slider').send_keys(Keys.LEFT)
-            # move to desicred result
-            for i in range((num_results//10)-1):
-                time.sleep(.25)
-                self.webdriver.find_element_by_id('result_slider').send_keys(Keys.RIGHT)
+                time.sleep(1)
 
-            time.sleep(1)
+                # set the region
+                try:
+                    self.webdriver.find_element_by_id('regionanchormore').click()
+                except WebDriverException as e:
+                    logger.warning('Regions probably already expanded.')
 
-            # save settings
-            self.webdriver.find_element_by_css_selector('#form-buttons div:first-child').click()
-            # accept alert
-            self.webdriver.switch_to.alert.accept()
+                region = self.config.get('google_selenium_region', 'US')
+                self.webdriver.find_element_by_css_selector('div[data-value="{}"]'.format(region)).click()
 
-            time.sleep(2)
+                # set the number of results
+                num_results = self.config.get('google_selenium_num_results', 10)
+                self.webdriver.find_element_by_id('result_slider').click()
+                # reset
+                for i in range(5):
+                    self.webdriver.find_element_by_id('result_slider').send_keys(Keys.LEFT)
+                # move to desicred result
+                for i in range((num_results//10)-1):
+                    time.sleep(.25)
+                    self.webdriver.find_element_by_id('result_slider').send_keys(Keys.RIGHT)
 
-            self.handle_request_denied()
+                time.sleep(1)
 
-        except WebDriverException as e:
-            logger.error(e)
-            raise e
+                # save settings
+                self.webdriver.find_element_by_css_selector('#form-buttons div:first-child').click()
+                # accept alert
+                self.webdriver.switch_to.alert.accept()
+
+                time.sleep(2)
+
+                self.handle_request_denied()
+
+            except WebDriverException as e:
+                logger.error(e)
+                raise e
 
 
 class DuckduckgoSelScrape(SelScrape):

README.md

@@ -2,14 +2,16 @@
 
 ## New News - August 2018
 
+For questions you can [contact me on my wegpage](https://incolumitas.com/) and write me an email there.
+
 This project is back to live after two years of abandonment. In the coming weeks, I will take some time to update all functionality to the most recent developments. This encompasses updating all Regexes and changes in search engine behavior. After a couple of weeks, you can expect this project to work again as documented here.
 
 #### Goals
 
 + Remove PhantomJS support **[Done]**
 + Maybe remove Firefox support **[Done: Decision was to keep Firefox]**
 + Think about using https://github.com/GoogleChrome/puppeteer instead of selenium for Chrome.
-+ Test all supported search engines
++ Test all supported search engines [Partly Done]
 
 ### Table of Contents