Merge pull request #436 from scholarly-python-package/develop

Release v1.7.2

Author: arunkannawadi

scholarly/_proxy_generator.py

@@ -185,7 +185,17 @@ def _use_proxy(self, http: str, https: str = None) -> bool:
             https = http
 
         proxies = {'http': http, 'https': https}
-        self._proxy_works = self._check_proxy(proxies)
+        if self.proxy_mode == ProxyMode.SCRAPERAPI:
+            r = requests.get("http://api.scraperapi.com/account", params={'api_key': self._API_KEY}).json()
+            if "error" in r:
+                self.logger.warning(r["error"])
+                self._proxy_works = False
+            else:
+                self._proxy_works = r["requestCount"] < int(r["requestLimit"])
+                self.logger.info("Successful ScraperAPI requests %d / %d",
+                                 r["requestCount"], r["requestLimit"])
+        else:
+            self._proxy_works = self._check_proxy(proxies)
 
         if self._proxy_works:
             self._session.proxies = proxies
@@ -562,6 +572,9 @@ def ScraperAPI(self, API_KEY, country_code=None, premium=False, render=False):
             self.logger.warning(r["error"])
             return False
 
+        self._API_KEY = API_KEY
+        self.proxy_mode = ProxyMode.SCRAPERAPI
+
         r["requestLimit"] = int(r["requestLimit"])
         self.logger.info("Successful ScraperAPI requests %d / %d",
                          r["requestCount"], r["requestLimit"])
@@ -586,7 +599,6 @@ def ScraperAPI(self, API_KEY, country_code=None, premium=False, render=False):
             proxy_works = self._use_proxy(http=f'{prefix}:{API_KEY}@proxy-server.scraperapi.com:8001')
             if proxy_works:
                 self.logger.info("ScraperAPI proxy setup successfully")
-                self.proxy_mode = ProxyMode.SCRAPERAPI
                 self._session.verify = False
                 return proxy_works
 

scholarly/_scholarly.py

@@ -15,8 +15,8 @@
 _AUTHSEARCH = '/citations?hl=en&view_op=search_authors&mauthors={0}'
 _KEYWORDSEARCH = '/citations?hl=en&view_op=search_authors&mauthors=label:{0}'
 _KEYWORDSEARCHBASE = '/citations?hl=en&view_op=search_authors&mauthors={}'
-_PUBSEARCH = '/scholar?hl=en&num=20&q={0}'
-_CITEDBYSEARCH = '/scholar?hl=en&num=20&cites={0}'
+_PUBSEARCH = '/scholar?hl=en&q={0}'
+_CITEDBYSEARCH = '/scholar?hl=en&cites={0}'
 _ORGSEARCH = "/citations?view_op=view_org&hl=en&org={0}"
 _MANDATES_URL = "https://scholar.google.com/citations?view_op=mandates_leaderboard_csv&hl=en"
 

scholarly/publication_parser.py

@@ -7,11 +7,11 @@
 
 _SCHOLARPUBRE = r'cites=([\d,]*)'
 _CITATIONPUB = '/citations?hl=en&view_op=view_citation&citation_for_view={0}'
-_SCHOLARPUB = '/scholar?hl=en&num=20&oi=bibs&cites={0}'
+_SCHOLARPUB = '/scholar?hl=en&oi=bibs&cites={0}'
 _CITATIONPUBRE = r'citation_for_view=([\w-]*:[\w-]*)'
-_BIBCITE = '/scholar?hl=en&num=20&q=info:{0}:scholar.google.com/\
+_BIBCITE = '/scholar?hl=en&q=info:{0}:scholar.google.com/\
 &output=cite&scirp={1}&hl=en'
-_CITEDBYLINK = '/scholar?hl=en&num=20&cites={0}'
+_CITEDBYLINK = '/scholar?hl=en&cites={0}'
 _MANDATES_URL = '/citations?view_op=view_mandate&hl=en&citation_for_view={0}'
 
 _BIB_MAPPING = {

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name='scholarly',
-    version='1.7.1',
+    version='1.7.2',
     author='Steven A. Cholewiak, Panos Ipeirotis, Victor Silva, Arun Kannawadi',
     author_email='steven@cholewiak.com, panos@stern.nyu.edu, vsilva@ualberta.ca, arunkannawadi@astro.princeton.edu',
     description='Simple access to Google Scholar authors and citations',