changed url_params to http_params for read_*

Author: Sky NSS

doc/source/whatsnew/v0.21.0.txt

@@ -86,29 +86,29 @@ Other Enhancements
 
 If `python-requests` library is installed try to use it first. If not, continue using urllib
 The :meth:`DataFrame.read_csv`, :meth:`DataFrame.read_html`, :meth:`DataFrame.read_json`,
-:meth:`DataFrame.read_excel` now allow optional param of ``url_params`` to pass in 
+:meth:`DataFrame.read_excel` now allow optional param of ``http_params`` to pass in 
 parameters for basic auth, disable ssl strict check or even a requests.Session() object
 
 
 .. ipython:: python
   import pandas as pd
 
-  # url_params is optional parameter. If it is non-empty, it attempts to use python-requests library
-  df = pd.read_csv('https://uname:[email protected]/bb.csv', url_params= {'auth': None} ) # now url can contain username and pwd
+  # http_params is optional parameter. If it is non-empty, it attempts to use python-requests library
+  df = pd.read_csv('https://uname:[email protected]/bb.csv', http_params= {'auth': None} ) # now url can contain username and pwd
   # Note - all basic auth scenarios require python-requests library
 
   # Basic Auth
-  df = pd.read_csv('https://aa.com/bb.csv', url_params={ 'auth': ('john', 'pwd') } ) # now url can contain username and pwd
+  df = pd.read_csv('https://aa.com/bb.csv', http_params={ 'auth': ('john', 'pwd') } ) # now url can contain username and pwd
 
   # Basic Auth And disable verification of SSL certificate eg: testing
   up = { 'auth': ('john', 'pwd') , 'verify' : False}
-  df = pd.read_csv('https://aa.com/bb.csv', url_params=up ) # now url can contain username and pwd
+  df = pd.read_csv('https://aa.com/bb.csv', http_params=up ) # now url can contain username and pwd
 
-  # Optionally, A requests.Session() can also be passed into url_params
+  # Optionally, A requests.Session() can also be passed into http_params
   import requests
   s = requests.Session()
   s.auth = MyAuthProvider('secret-key') # custom auth provider supported by requests
-  df = pd.read_csv(url, url_params=s)
+  df = pd.read_csv(url, http_params=s)
 
   # For advanced users, this may provide extensibility. However, testing on pandas side is limited to basic scenarios
   # here is an example of advanced scenario
@@ -119,14 +119,14 @@ parameters for basic auth, disable ssl strict check or even a requests.Session()
   s.headers.update( {'User-Agent': 'Custom user agent'} )  # extensible to set any custom header needed
   s.proxies = { 'http': 'http://a.com:100'}  # if user has proxies 
   s.cert = '/path/client.cert'                     # if custom cert is needed
-  df = pd.read_csv( 'https://aa.com/bbb.csv', url_params=s)
+  df = pd.read_csv( 'https://aa.com/bbb.csv', http_params=s)
 
   def print_http_status(r, *args, **kwargs):
       print(r.status_code)
       print(r.headers['Content-Length'])
   s = Session()
   s.hooks = dict(response=print_http_status)
-  df = pd.read_csv( 'https://aa.com/bbb.csv', url_params=s)
+  df = pd.read_csv( 'https://aa.com/bbb.csv', http_params=s)
 
 
 .. _whatsnew_0210.api_breaking:

pandas/io/common.py

@@ -191,29 +191,29 @@ def _is_handled_by_requests(o):
     return _is_url(o) and parse_url(o).scheme in ['http', 'https']
 
 
-def gen_session(url_params):
+def gen_session(http_params):
     """
-    Generate python-requests session from url_params dict
+    Generate python-requests session from http_params dict
     """
     s = None
-    if url_params and type(url_params) is requests.sessions.Session:
-        s = url_params
+    if http_params and type(http_params) is requests.sessions.Session:
+        s = http_params
     else:
         s = requests.Session()
         s.stream = True
         # Setting accept-encoding to None for backwards compatibility with
         # urlopen. ideally we want to allow gzip download
         # urlopen doesnt decompress automatically, requests does.
         s.headers.update({'Accept-Encoding': None})
-    if url_params and type(url_params) is dict:
-        if url_params.get('auth', None) and not s.auth:
-            s.auth = url_params.get('auth')
-        if url_params.get('verify', True) is False and s.verify is not False:
-            s.verify = url_params.get('verify')
+    if http_params and type(http_params) is dict:
+        if http_params.get('auth', None) and not s.auth:
+            s.auth = http_params.get('auth')
+        if http_params.get('verify', True) is False and s.verify is not False:
+            s.verify = http_params.get('verify')
     return s
 
 
-def fetch_url(url, url_params=None, skip_requests=False):
+def fetch_url(url, http_params=None, skip_requests=False):
     """
     If url is url, first try python-requests else try urllib.
     Note if requests library is used, auto gunzip is
@@ -226,7 +226,7 @@ def fetch_url(url, url_params=None, skip_requests=False):
             'http://cnn.com'
             'file:///home/sky/aaa.csv'
 
-    url_params : dict or requests.Session(), default None
+    http_params : dict or requests.Session(), default None
         A python dict containing:
             'auth': tuple (str, str) eg (unae, pwd)
             'auth': Any other auth object accepted by requests
@@ -244,20 +244,20 @@ def fetch_url(url, url_params=None, skip_requests=False):
         .. versionadded:: 0.21.0
     Raises
     ------
-    ValueError if url_params specified without installed python-requests pkg
+    ValueError if http_params specified without installed python-requests pkg
     """
-    if not url_params:
+    if not http_params:
         skip_requests = True
     if (not skip_requests) and \
             is_requests_pkg_avail() and \
             _is_handled_by_requests(url):
-        s = gen_session(url_params)
+        s = gen_session(http_params)
         resp = s.get(url)
         resp.raise_for_status()
         content_bytes = resp.content
     else:
-        if url_params and (skip_requests or not is_requests_pkg_avail()):
-            msg = 'To utilize url_params, python-requests library is ' + \
+        if http_params and (skip_requests or not is_requests_pkg_avail()):
+            msg = 'To utilize http_params, python-requests library is ' + \
                   'required but not detected'
             raise ValueError(msg)
         resp = _urlopen(url)
@@ -266,7 +266,7 @@ def fetch_url(url, url_params=None, skip_requests=False):
 
 
 def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
-                           compression=None, url_params=None,
+                           compression=None, http_params=None,
                            skip_requests=False):
     """
     If the filepath_or_buffer is a url, translate and return the buffer.
@@ -281,7 +281,7 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
     compression : str, default None
         indicate the compression such as 'gzip'.
 
-    url_params : dict or requests.Session(), default None
+    http_params : dict or requests.Session(), default None
         A python dict containing:
             'auth': tuple (str, str) eg (unae, pwd)
             'auth': Any other auth object accepted by requests
@@ -304,13 +304,13 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
 
     Raises
     ------
-    ValueError if url_params specified without installed python-requests pkg
+    ValueError if http_params specified without installed python-requests pkg
     """
     filepath_or_buffer = _stringify_path(filepath_or_buffer)
 
     if _is_url(filepath_or_buffer):
         req, content_bytes = fetch_url(filepath_or_buffer,
-                                       url_params,
+                                       http_params,
                                        skip_requests)
         reader = BytesIO(content_bytes)
         content_encoding = req.headers.get('Content-Encoding', None)

pandas/io/excel.py

@@ -213,7 +213,7 @@ def read_excel(io, sheet_name=0, header=0, skiprows=None, skip_footer=0,
 
     if not isinstance(io, ExcelFile):
         io = ExcelFile(io, engine=engine,
-                       url_params=kwds.get('url_params', None))
+                       http_params=kwds.get('http_params', None))
 
     return io._parse_excel(
         sheetname=sheet_name, header=header, skiprows=skiprows, names=names,
@@ -261,8 +261,8 @@ def __init__(self, io, **kwds):
         # If io is a url, want to keep the data as bytes so can't pass
         # to get_filepath_or_buffer()
         if _is_url(self.io):
-            rs = kwds.get('url_params', None)
-            req, content = fetch_url(self.io, url_params=rs)
+            rs = kwds.get('http_params', None)
+            req, content = fetch_url(self.io, http_params=rs)
             io = BytesIO(content)
         elif not isinstance(self.io, (ExcelFile, xlrd.Book)):
             io, _, _ = get_filepath_or_buffer(self._io)

pandas/io/html.py

@@ -116,14 +116,14 @@ def _get_skiprows(skiprows):
                     type(skiprows).__name__)
 
 
-def _read(obj, url_params=None):
+def _read(obj, http_params=None):
     """Try to read from a url, file or string.
 
     Parameters
     ----------
     obj : str, unicode, or file-like
 
-    url_params : dict or requests.Session(), default None
+    http_params : dict or requests.Session(), default None
         A python dict containing:
             'auth': tuple (str, str) eg (unae, pwd)
             'auth': Any other auth object accepted by requests
@@ -140,7 +140,7 @@ def _read(obj, url_params=None):
     raw_text : str
     """
     if _is_url(obj):
-        req, text = fetch_url(obj, url_params)
+        req, text = fetch_url(obj, http_params)
     elif hasattr(obj, 'read'):
         text = obj.read()
     elif isinstance(obj, char_types):
@@ -198,12 +198,12 @@ class _HtmlFrameParser(object):
     functionality.
     """
 
-    def __init__(self, io, match, attrs, encoding, url_params=None):
+    def __init__(self, io, match, attrs, encoding, http_params=None):
         self.io = io
         self.match = match
         self.attrs = attrs
         self.encoding = encoding
-        self.url_params = url_params
+        self.http_params = http_params
 
     def parse_tables(self):
         tables = self._parse_tables(self._build_doc(), self.match, self.attrs)
@@ -456,7 +456,7 @@ def _parse_tables(self, doc, match, attrs):
         return result
 
     def _setup_build_doc(self):
-        raw_text = _read(self.io, self.url_params)
+        raw_text = _read(self.io, self.http_params)
         if not raw_text:
             raise ValueError('No text parsed from document: %s' % self.io)
         return raw_text
@@ -744,7 +744,7 @@ def _parse(flavor, io, match, attrs, encoding, **kwargs):
     for flav in flavor:
         parser = _parser_dispatch(flav)
         p = parser(io, compiled_match, attrs, encoding,
-                   url_params=kwargs.get('url_params', None))
+                   http_params=kwargs.get('http_params', None))
 
         try:
             tables = p.parse_tables()
@@ -768,7 +768,7 @@ def read_html(io, match='.+', flavor=None, header=None, index_col=None,
               skiprows=None, attrs=None, parse_dates=False,
               tupleize_cols=False, thousands=',', encoding=None,
               decimal='.', converters=None, na_values=None,
-              keep_default_na=True, url_params=None):
+              keep_default_na=True, http_params=None):
     r"""Read HTML tables into a ``list`` of ``DataFrame`` objects.
 
     Parameters
@@ -869,7 +869,7 @@ def read_html(io, match='.+', flavor=None, header=None, index_col=None,
 
         .. versionadded:: 0.19.0
 
-    url_params : requests.Session(), default None
+    http_params : requests.Session(), default None
         A python requests.Session object if http(s) path to enable basic auth
         and many other scenarios that requests allows
 
@@ -922,4 +922,4 @@ def read_html(io, match='.+', flavor=None, header=None, index_col=None,
                   parse_dates=parse_dates, tupleize_cols=tupleize_cols,
                   thousands=thousands, attrs=attrs, encoding=encoding,
                   decimal=decimal, converters=converters, na_values=na_values,
-                  keep_default_na=keep_default_na, url_params=url_params)
+                  keep_default_na=keep_default_na, http_params=http_params)

pandas/io/json/json.py

@@ -174,7 +174,7 @@ def write(self):
 def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
               convert_axes=True, convert_dates=True, keep_default_dates=True,
               numpy=False, precise_float=False, date_unit=None, encoding=None,
-              lines=False, url_params=None):
+              lines=False, http_params=None):
     """
     Convert a JSON string to pandas object
 
@@ -263,7 +263,7 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
 
         .. versionadded:: 0.19.0
 
-    url_params : dict or requests.Session(), default None
+    http_params : dict or requests.Session(), default None
         A python dict containing:
             'auth': tuple (str, str) eg (unae, pwd)
             'auth': Any other auth object accepted by requests
@@ -334,7 +334,7 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
 
     filepath_or_buffer, _, _ = get_filepath_or_buffer(path_or_buf,
                                                       encoding=encoding,
-                                                      url_params=url_params)
+                                                      http_params=http_params)
     if isinstance(filepath_or_buffer, compat.string_types):
         try:
             exists = os.path.exists(filepath_or_buffer)

pandas/io/parsers.py

@@ -391,11 +391,11 @@ def _read(filepath_or_buffer, kwds):
         encoding = re.sub('_', '-', encoding).lower()
         kwds['encoding'] = encoding
 
-    url_params = kwds.get('url_params', None)
+    http_params = kwds.get('http_params', None)
     compression = kwds.get('compression')
     compression = _infer_compression(filepath_or_buffer, compression)
     filepath_or_buffer, _, compression = get_filepath_or_buffer(
-        filepath_or_buffer, encoding, compression, url_params)
+        filepath_or_buffer, encoding, compression, http_params)
     kwds['compression'] = compression
 
     if kwds.get('date_parser', None) is not None:
@@ -579,7 +579,7 @@ def parser_f(filepath_or_buffer,
                  float_precision=None,
 
                  # python requests session
-                 url_params=None,
+                 http_params=None,
                  ):
 
         # Alias sep -> delimiter.
@@ -662,7 +662,7 @@ def parser_f(filepath_or_buffer,
                     infer_datetime_format=infer_datetime_format,
                     skip_blank_lines=skip_blank_lines,
 
-                    url_params=url_params)
+                    http_params=http_params)
 
         return _read(filepath_or_buffer, kwds)
 

pandas/tests/io/test_http_auth.py

@@ -142,18 +142,18 @@ def check_http_auth(url, uname, pwd, verify_ssl):
 
     def get_df(url, uname, pwd, verify_ssl, pd_read_fn, fname):
         furl = url + fname
-        url_params = {}
+        http_params = {}
         if uname or pwd:
-            url_params['auth'] = (uname, pwd)
+            http_params['auth'] = (uname, pwd)
         if verify_ssl is not None:
-            url_params['verify'] = verify_ssl
+            http_params['verify'] = verify_ssl
         msg = '{0: <90} -- auth:[{1: <10}/{2: <10}] v:[{3: <5}]'.format(
               furl, str(uname), str(pwd), str(verify_ssl))
         if verify_ssl or furl.lower().startswith('http://'):
-            df = pd_read_fn(furl, url_params=url_params)
+            df = pd_read_fn(furl, http_params=http_params)
         else:
             with tm.assert_produces_warning(InsecureRequestWarning):
-                df = pd_read_fn(furl, url_params=url_params)
+                df = pd_read_fn(furl, http_params=http_params)
         if type(df) is list:  # html
             df = df[0]
         smatch = match_csv(df)
@@ -184,9 +184,9 @@ def get_df(url, uname, pwd, verify_ssl, pd_read_fn, fname):
                              {'auth': ('uname', 'pwd'), 'verify': False},
                          ]
                          )
-def test_url_params(up):
+def test_http_params(up):
     _skip_if_no_requests()
-    s = pd.io.common.gen_session(url_params=up)
+    s = pd.io.common.gen_session(http_params=up)
     assert type(s) is requests.sessions.Session
     if up and up.get('auth', None):
         assert s.auth == up.get('auth', None)
@@ -199,7 +199,7 @@ def test_pass_session_obj():
     s = requests.sessions.Session()
     s.auth = ('uname', 'pwd')
     s.verify = False
-    t = pd.io.common.gen_session(url_params=s)
+    t = pd.io.common.gen_session(http_params=s)
     assert s == t
     assert s.auth == t.auth
     assert s.verify == t.verify
@@ -209,5 +209,5 @@ def test_skip_requests():
     with pytest.raises(ValueError):
         a = (uname, pwd)
         resp, content_bytes = pd.io.common.fetch_url('http://cnn.com',
-                                                     url_params={'auth': a},
+                                                     http_params={'auth': a},
                                                      skip_requests=True)