Updates to comments, etc + some change in username password logic

Author: Sky NSS

doc/source/whatsnew/v0.21.0.txt

@@ -40,8 +40,8 @@ Other Enhancements
 - :func:`DataFrame.clip()` and :func:`Series.clip()` have gained an ``inplace`` argument. (:issue:`15388`)
 - :func:`crosstab` has gained a ``margins_name`` parameter to define the name of the row / column that will contain the totals when ``margins=True``. (:issue:`15972`)
 - :func:`Dataframe.select_dtypes` now accepts scalar values for include/exclude as well as list-like. (:issue:`16855`)
-- :func:`read_csv` `read_html` `read_json` `read_html`  now accept auth in url //<user>:<password>@<host>:<port>/<url-path>, or ``auth`` tuple (username, password) parameter 
-- :func:`read_csv` `read_html` `read_json` `read_html`  now accept ``verify_ssl`` False to disable https/ssl certificate verification (eg: self signed ssl certs in testing) 
+- :func:`read_csv`, :func:`read_html`, :func:`read_json`, :func:`read_html`  now accept auth in url //<user>:<password>@<host>:<port>/<url-path>, or ``auth`` tuple (username, password) parameter 
+- :func:`read_csv`, :func:`read_html`, :func:`read_json`, :func:`read_html`  now accept ``verify_ssl`` False to disable https/ssl certificate verification (eg: self signed ssl certs in testing)  (:issue:`16716`)
 .. _whatsnew_0210.api_breaking:
 
 Backwards incompatible API changes

pandas/io/common.py

@@ -190,12 +190,27 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
     ----------
     filepath_or_buffer : a url, filepath (str, py.path.local or pathlib.Path),
                          or buffer
-            supports 'https://username:[email protected]:port/aaa.csv'
+            now supports 'https://<user>:<password>@<host>:<port>/<url-path>'
+            
+            .. versionadded:: 0.21.0
+            
     encoding : the encoding to use to decode py3 bytes, default is 'utf-8'
-    compression:
-    auth: (str,str), default None. (username, password) for HTTP(s) basic auth
-    verify_ssl: Default True. If False, allow self signed and invalid SSL
-                 certificates for https
+
+    compression : string, default None
+    
+            .. versionadded:: 0.18.1
+    
+    auth : tuple, default None 
+            A tuple of string with (username, password) string for 
+            HTTP(s) basic auth: eg auth= ('roberto', 'panda$4life')
+            
+            .. versionadded:: 0.21.0
+
+    verify_ssl : boolean, Default True
+            If False, allow self signed and invalid SSL certificates for https
+
+            .. versionadded:: 0.21.0
+
 
     Returns
     -------
@@ -263,19 +278,34 @@ def split_auth_from_url(url_with_uname):
 
     Parameters
     ----------
-    url_with_uname : a url that may or may not contain username and password
+    url_with_uname : string
+            a url that may or may not contain username and password
             see section 3.1 RFC 1738 https://www.ietf.org/rfc/rfc1738.txt
             //<user>:<password>@<host>:<port>/<url-path>
+            
+            .. versionadded:: 0.21.0
 
     Returns
     -------
-    (username, password), url_no_usrpwd : username or "", password or "", 
-         url without username or password (if it contained it )
+    (username, password), url_no_usrpwd : tuple, string  Default ('', '') url
+            A tuple with (username, pwd) pair and 
+            url without username or password (if it contained it )
+
+    Raises
+    ------
+    ValueError for empty url
     """
+    if not url_with_uname: 
+        msg = "Empty url: {_type}"
+        raise ValueError(msg.format(_type=type(url_with_uname)))
     o = parse_url(url_with_uname)
-    usrch = '{}:{}@{}'.format(o.username, o.password, o.hostname)
-    url_no_usrpwd = url_with_uname.replace(usrch, o.hostname)
-    return (o.username, o.password), url_no_usrpwd
+    uname = o.username if o.username else ''
+    pwd = o.password if o.password else ''
+    url_no_usrpwd = url_with_uname
+    if uname or pwd:
+        usrch = '{}:{}@{}'.format(o.username, o.password, o.hostname)
+        url_no_usrpwd = url_with_uname.replace(usrch, o.hostname)
+    return (uname, pwd), url_no_usrpwd
 
 
 def get_urlopen_args(url_with_uname, auth=None, verify_ssl=True):
@@ -286,30 +316,42 @@ def get_urlopen_args(url_with_uname, auth=None, verify_ssl=True):
 
     Parameters
     ----------
-    url_with_uname : a url that may or may not contain username and password
+    url_with_uname : string
+            a url that may or may not contain username and password
             see section 3.1 RFC 1738 https://www.ietf.org/rfc/rfc1738.txt
             //<user>:<password>@<host>:<port>/<url-path>
-    auth : ( username/""/None, password/"", None) tuple
-    verify_ssl: If False, SSL certificate verification is disabled.
+            
+            .. versionadded:: 0.21.0
+
+    auth : tuple, default None 
+            A tuple of string with (username, password) string for 
+            HTTP(s) basic auth: eg auth= ('roberto', 'panda$4life')
+            
+            .. versionadded:: 0.21.0
+
+    verify_ssl : boolean, Default True
+            If False, allow self signed and invalid SSL certificates for https
+
+            .. versionadded:: 0.21.0
 
     Returns
     -------
     Request, kwargs to pass to urlopen. kwargs may be {} or {'context': obj }
     """
     uname = pwd = None
+    url_no_usrpwd = url_with_uname
     if auth and len(auth) == 2:
         uname, pwd = auth
     if not uname and not pwd:
         (uname, pwd), url_no_usrpwd = split_auth_from_url(url_with_uname)
-    else:
-        url_no_usrpwd = url_with_uname
-    upstr = '{}:{}'.format(uname, pwd)
-    if compat.PY3:
-        b64str = base64.b64encode(bytes(upstr, 'ascii')).decode('utf-8')
-    else:
-        b64str = base64.encodestring(upstr).replace('\n', '')
     req = Request(url_no_usrpwd)
-    req.add_header("Authorization", "Basic {}".format(b64str))
+    if uname or pwd:
+        upstr = '{}:{}'.format(uname, pwd)
+        if compat.PY3:
+            b64str = base64.b64encode(bytes(upstr, 'ascii')).decode('utf-8')
+        else:
+            b64str = base64.encodestring(upstr).replace('\n', '')
+        req.add_header("Authorization", "Basic {}".format(b64str))
     kwargs = {}
     if verify_ssl not in [None, True]:
         kwargs['context'] = ssl._create_unverified_context()

pandas/io/html.py

@@ -123,8 +123,16 @@ def _read(obj, auth=None, verify_ssl=None):
     Parameters
     ----------
     obj : str, unicode, or file-like
-    auth: None or (username, password) for http basic auth
-    verify_ssl: Default True. Set to False to disable cert verification
+    auth : tuple, default None 
+            A tuple of string with (username, password) string for 
+            HTTP(s) basic auth: eg auth= ('roberto', 'panda$4life')
+            
+            .. versionadded:: 0.21.0
+
+    verify_ssl : boolean, Default True
+            If False, allow self signed and invalid SSL certificates for https
+
+            .. versionadded:: 0.21.0
     Returns
     -------
     raw_text : str
@@ -866,10 +874,16 @@ def read_html(io, match='.+', flavor=None, header=None, index_col=None,
 
         .. versionadded:: 0.19.0
 
-    auth: (str,str), default None. (username, password) for HTTP(s) basic auth
+    auth : tuple, default None 
+            A tuple of string with (username, password) string for 
+            HTTP(s) basic auth: eg auth= ('roberto', 'panda$4life')
+            
+            .. versionadded:: 0.21.0
+
+    verify_ssl : boolean, Default True
+            If False, allow self signed and invalid SSL certificates for https
 
-    verify_ssl : bool, default True
-        If False, ssl certificate is not verified (allow self signed SSL certs)
+            .. versionadded:: 0.21.0
 
         Returns
     -------

pandas/io/json/json.py

@@ -263,9 +263,16 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
 
         .. versionadded:: 0.19.0
 
-    auth: (str,str), default None. (username, password) for HTTP(s) basic auth
-    verify_ssl: boolean, default None (True).
-                    If false, allow self siged SSL certificates
+    auth : tuple, default None 
+            A tuple of string with (username, password) string for 
+            HTTP(s) basic auth: eg auth= ('roberto', 'panda$4life')
+            
+            .. versionadded:: 0.21.0
+
+    verify_ssl : boolean, Default True
+            If False, allow self signed and invalid SSL certificates for https
+
+            .. versionadded:: 0.21.0
 
     Returns
     -------

pandas/io/parsers.py

@@ -579,10 +579,8 @@ def parser_f(filepath_or_buffer,
                  memory_map=False,
                  float_precision=None,
 
-                 # Basic auth (http/https) (username, password)
                  auth=None,
 
-                 # skip verify self signed SSL certificates
                  verify_ssl=None):
 
         # Alias sep -> delimiter.

pandas/tests/test_common.py

@@ -189,17 +189,24 @@ def test_write_fspath_hdf5(self):
             expected = pd.read_hdf(string, key='bar')
 
         tm.assert_frame_equal(result, expected)
-
-    def test_split_url_extract_uname_pwd(self):
-        for url, uname, pwd, nurl in [('https://aaa:[email protected]:1010/aaa.txt',
-                                       'aaa',
-                                       'bbb',
-                                       'https://ccc.com:1010/aaa.txt'
-                                       )]:
-            un, p, u = common.split_uname_from_url(url)
-            assert u == nurl
-            assert un == uname
-            assert p == pwd
+    
+    @pytest.mark.parametrize('url, uname, pwd, nurl', [
+        ('https://a1:[email protected]:101/f.csv',
+         'aaa',
+         'bbb',
+         'https://cc.com:101/f.csv'
+        ),
+        ('https://ccc.com:1010/aaa.txt',
+         '',
+         '',
+         'https://ccc.com:1010/aaa.txt'
+        ),
+    ])
+    def test_split_url_extract_uname_pwd(self, url, uname, pwd, nurl):
+        un, pw, ur = common.split_uname_from_url(url)
+        assert ur == nurl
+        assert un == uname
+        assert pw == pwd
 
 
 class TestMMapWrapper(object):