Merge pull request #478 from scholarly-python-package/develop

Release v1.7.8

Author: arunkannawadi

.github/workflows/proxytests.yml

@@ -4,7 +4,7 @@
 name: Python package (with methods that need proxy)
 
 on:
-  pull_request:
+  push:
     branches: [main]
 
 jobs:
@@ -31,7 +31,6 @@ jobs:
       #  uses: typilus/typilus-action@v0.9
       - name: Run unittests
         id: unittests
-        if: ${{ github.event.pull_request.ready_for_review == true }}
         continue-on-error: true
         env:
           CONNECTION_METHOD: ${{ secrets.CONNECTION_METHOD }}

scholarly/_navigator.py

@@ -74,12 +74,12 @@ def use_proxy(self, pg1: ProxyGenerator, pg2: ProxyGenerator = None):
         self._session1 = self.pm1.get_session()
         self._session2 = self.pm2.get_session()
 
-    def _new_session(self, premium=True):
+    def _new_session(self, premium=True, **kwargs):
         self.got_403 = False
         if premium:
-            self._session1 = self.pm1._new_session()
+            self._session1 = self.pm1._new_session(**kwargs)
         else:
-            self._session2 = self.pm2._new_session()
+            self._session2 = self.pm2._new_session(**kwargs)
 
 
     def _get_page(self, pagerequest: str, premium: bool = False) -> str:
@@ -112,7 +112,8 @@ def _get_page(self, pagerequest: str, premium: bool = False) -> str:
                 w = random.uniform(1,2)
                 time.sleep(w)
                 resp = session.get(pagerequest, timeout=timeout)
-                self.logger.debug("Session proxy config is {}".format(pm._proxies))
+                if premium is False:  # premium methods may contain sensitive information
+                    self.logger.debug("Session proxy config is {}".format(pm._proxies))
 
                 has_captcha = self._requests_has_captcha(resp.text)
 

scholarly/_proxy_generator.py

@@ -183,11 +183,11 @@ def _use_proxy(self, http: str, https: str = None) -> bool:
         :returns: whether or not the proxy was set up successfully
         :rtype: {bool}
         """
-        if https is None:
-            https = http
         if http[:4] != "http":
             http = "http://" + http
-        if https[:5] != "https":
+        if https is None:
+            https = http
+        elif https[:5] != "https":
             https = "https://" + https
 
         proxies = {'http://': http, 'https://': https}
@@ -205,7 +205,7 @@ def _use_proxy(self, http: str, https: str = None) -> bool:
 
         if self._proxy_works:
             self._proxies = proxies
-            self._new_session()
+            self._new_session(proxies=proxies)
 
         return self._proxy_works
 
@@ -444,8 +444,9 @@ def _handle_captcha2(self, url):
 
         return self._session
 
-    def _new_session(self):
+    def _new_session(self, **kwargs):
         init_kwargs = {}
+        init_kwargs.update(kwargs)
         proxies = {}
         if self._session:
             proxies = self._proxies
@@ -612,8 +613,9 @@ def ScraperAPI(self, API_KEY, country_code=None, premium=False, render=False):
         for _ in range(3):
             proxy_works = self._use_proxy(http=f'{prefix}:{API_KEY}@proxy-server.scraperapi.com:8001')
             if proxy_works:
+                proxies = {'http://': f"http://scraperapi:{API_KEY}@proxy-server.scraperapi.com:8001",}
                 self.logger.info("ScraperAPI proxy setup successfully")
-                self._session.verify = False
+                self._new_session(verify=False, proxies=proxies)
                 return proxy_works
 
         if (r["requestCount"] >= r["requestLimit"]):

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name='scholarly',
-    version='1.7.7',
+    version='1.7.8',
     author='Steven A. Cholewiak, Panos Ipeirotis, Victor Silva, Arun Kannawadi',
     author_email='steven@cholewiak.com, panos@stern.nyu.edu, vsilva@ualberta.ca, arunkannawadi@astro.princeton.edu',
     description='Simple access to Google Scholar authors and citations',