Added http(s) basic auth and allow self signed ssl certs

Author: Sky NSS

pandas/io/common.py

@@ -4,6 +4,8 @@
 import csv
 import codecs
 import mmap
+import ssl
+import base64
 from contextlib import contextmanager, closing
 
 from pandas.compat import StringIO, BytesIO, string_types, text_type
@@ -49,7 +51,11 @@
 
 
 if compat.PY3:
-    from urllib.request import urlopen, pathname2url
+    from urllib.request import (urlopen, pathname2url, build_opener,
+                                install_opener,
+                                HTTPPasswordMgrWithDefaultRealm,
+                                HTTPBasicAuthHandler,
+                                HTTPSHandler)
     _urlopen = urlopen
     from urllib.parse import urlparse as parse_url
     from urllib.parse import (uses_relative, uses_netloc, uses_params,
@@ -58,6 +64,7 @@
     from http.client import HTTPException  # noqa
 else:
     from urllib2 import urlopen as _urlopen
+    from urllib2 import Request
     from urllib import urlencode, pathname2url  # noqa
     from urlparse import urlparse as parse_url
     from urlparse import uses_relative, uses_netloc, uses_params, urljoin
@@ -177,7 +184,8 @@ def _stringify_path(filepath_or_buffer):
 
 
 def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
-                           compression=None):
+                           compression=None, username=None,
+                           password=None, verify_ssl=None):
     """
     If the filepath_or_buffer is a url, translate and return the buffer.
     Otherwise passthrough.
@@ -186,7 +194,13 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
     ----------
     filepath_or_buffer : a url, filepath (str, py.path.local or pathlib.Path),
                          or buffer
+            support 'https://username:[email protected]:port/aaa.csv'
     encoding : the encoding to use to decode py3 bytes, default is 'utf-8'
+    compression:
+    username: Authentication username (for https basic auth)
+    password: Authentication password (for https basic auth)
+    verify_ssl: Default True. If False, allow self signed and invalid SSL
+                 certificates for https
 
     Returns
     -------
@@ -195,7 +209,11 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
     filepath_or_buffer = _stringify_path(filepath_or_buffer)
 
     if _is_url(filepath_or_buffer):
-        req = _urlopen(filepath_or_buffer)
+        ureq, kwargs = get_urlopen_args(filepath_or_buffer,
+                                        uname=username,
+                                        pwd=password,
+                                        verify_ssl=verify_ssl)
+        req = _urlopen(ureq, **kwargs)
         content_encoding = req.headers.get('Content-Encoding', None)
         if content_encoding == 'gzip':
             # Override compression based on Content-Encoding header
@@ -244,6 +262,53 @@ def file_path_to_url(path):
 }
 
 
+def split_uname_from_url(url_with_uname):
+    o = parse_url(url_with_uname)
+    usrch = '{}:{}@{}'.format(o.username, o.password, o.hostname)
+    url_no_usrpwd = url_with_uname.replace(usrch, o.hostname)
+    return o.username, o.password, url_no_usrpwd
+
+
+def get_urlopen_args(url_with_uname, uname=None, pwd=None, verify_ssl=True):
+    if not uname and not pwd:
+        uname, pwd, url_no_usrpwd = split_uname_from_url(url_with_uname)
+    else:
+        url_no_usrpwd = url_with_uname
+    if compat.PY3:
+        fn = get_urlopen_args_py3
+    else:
+        fn = get_urlopen_args_py2
+    req, kwargs = fn(uname, pwd, url_no_usrpwd, verify_ssl=verify_ssl)
+    return req, kwargs
+
+
+def get_urlopen_args_py2(uname, pwd, url_no_usrpwd, verify_ssl=True):
+    req = Request(url_no_usrpwd)
+    upstr = '{}:{}'.format(uname, pwd)
+    base64string = base64.encodestring(upstr).replace('\n', '')
+    req.add_header("Authorization", "Basic {}".format(base64string))
+    # I hope pandas can support self signed certs too
+    kwargs = {}
+    if verify_ssl not in [None, True]:
+        kwargs['context'] = ssl._create_unverified_context()
+    return req, kwargs
+
+
+def get_urlopen_args_py3(uname, pwd, url_no_usrpwd, verify_ssl=True):
+    passman = HTTPPasswordMgrWithDefaultRealm()
+    passman.add_password(None, url_no_usrpwd, uname, pwd)
+    authhandler = HTTPBasicAuthHandler(passman)
+    if verify_ssl in [None, True]:
+        opener = build_opener(authhandler)
+    else:
+        context = ssl.create_default_context()
+        context.check_hostname = False
+        context.verify_mode = ssl.CERT_NONE
+        opener = build_opener(authhandler, HTTPSHandler(context=context))
+    install_opener(opener)
+    return url_no_usrpwd, {}
+
+
 def _infer_compression(filepath_or_buffer, compression):
     """
     Get the compression method for filepath_or_buffer. If compression='infer',

pandas/io/excel.py

@@ -20,7 +20,7 @@
 from pandas.errors import EmptyDataError
 from pandas.io.common import (_is_url, _urlopen, _validate_header_arg,
                               get_filepath_or_buffer, _NA_VALUES,
-                              _stringify_path)
+                              _stringify_path, get_urlopen_args)
 from pandas.core.indexes.period import Period
 import pandas._libs.json as json
 from pandas.compat import (map, zip, reduce, range, lrange, u, add_metaclass,
@@ -200,7 +200,6 @@ def read_excel(io, sheet_name=0, header=0, skiprows=None, skip_footer=0,
                convert_float=True, converters=None, dtype=None,
                true_values=None, false_values=None, engine=None,
                squeeze=False, **kwds):
-
     # Can't use _deprecate_kwarg since sheetname=None has a special meaning
     if is_integer(sheet_name) and sheet_name == 0 and 'sheetname' in kwds:
         warnings.warn("The `sheetname` keyword is deprecated, use "
@@ -211,7 +210,11 @@ def read_excel(io, sheet_name=0, header=0, skiprows=None, skip_footer=0,
                         "Use just `sheet_name`")
 
     if not isinstance(io, ExcelFile):
-        io = ExcelFile(io, engine=engine)
+        io = ExcelFile(io,
+                       engine=engine,
+                       username=kwds.get('username', None),
+                       password=kwds.get('password', None),
+                       verify_ssl=kwds.get('verify_ssl', None))
 
     return io._parse_excel(
         sheetname=sheet_name, header=header, skiprows=skiprows, names=names,
@@ -259,7 +262,12 @@ def __init__(self, io, **kwds):
         # If io is a url, want to keep the data as bytes so can't pass
         # to get_filepath_or_buffer()
         if _is_url(self._io):
-            io = _urlopen(self._io)
+            verify_ssl = kwds.get('verify_ssl', None)
+            ureq, kwargs = get_urlopen_args(self._io,
+                                            uname=kwds.get('username', None),
+                                            pwd=kwds.get('password', None),
+                                            verify_ssl=verify_ssl)
+            io = _urlopen(ureq, **kwargs)
         elif not isinstance(self.io, (ExcelFile, xlrd.Book)):
             io, _, _ = get_filepath_or_buffer(self._io)
 

pandas/io/html.py

@@ -15,7 +15,8 @@
 from pandas.core.dtypes.common import is_list_like
 from pandas.errors import EmptyDataError
 from pandas.io.common import (_is_url, urlopen,
-                              parse_url, _validate_header_arg)
+                              parse_url, _validate_header_arg,
+                              get_urlopen_args)
 from pandas.io.parsers import TextParser
 from pandas.compat import (lrange, lmap, u, string_types, iteritems,
                            raise_with_traceback, binary_type)
@@ -116,19 +117,22 @@ def _get_skiprows(skiprows):
                     type(skiprows).__name__)
 
 
-def _read(obj):
+def _read(obj, username=None, password=None, verify_ssl=None):
     """Try to read from a url, file or string.
 
     Parameters
     ----------
     obj : str, unicode, or file-like
-
+    username: username for http basic auth
+    password: password for http basic auth
+    verify_ssl: Default True. Set to False to disable cert verification
     Returns
     -------
     raw_text : str
     """
     if _is_url(obj):
-        with urlopen(obj) as url:
+        ureq, kwargs = get_urlopen_args(obj, username, password, verify_ssl)
+        with urlopen(ureq, **kwargs) as url:
             text = url.read()
     elif hasattr(obj, 'read'):
         text = obj.read()
@@ -187,11 +191,15 @@ class _HtmlFrameParser(object):
     functionality.
     """
 
-    def __init__(self, io, match, attrs, encoding):
+    def __init__(self, io, match, attrs, encoding, username=None,
+                 password=None, verify_ssl=None):
         self.io = io
         self.match = match
         self.attrs = attrs
         self.encoding = encoding
+        self.username = username
+        self.password = password
+        self.verify_ssl = verify_ssl
 
     def parse_tables(self):
         tables = self._parse_tables(self._build_doc(), self.match, self.attrs)
@@ -444,7 +452,8 @@ def _parse_tables(self, doc, match, attrs):
         return result
 
     def _setup_build_doc(self):
-        raw_text = _read(self.io)
+        raw_text = _read(self.io, self.username,
+                         self.password, self.verify_ssl)
         if not raw_text:
             raise ValueError('No text parsed from document: %s' % self.io)
         return raw_text
@@ -731,8 +740,12 @@ def _parse(flavor, io, match, attrs, encoding, **kwargs):
     retained = None
     for flav in flavor:
         parser = _parser_dispatch(flav)
-        p = parser(io, compiled_match, attrs, encoding)
-
+        p = parser(io, compiled_match,
+                   attrs,
+                   encoding,
+                   username=kwargs.get('username', None),
+                   password=kwargs.get('password', None),
+                   verify_ssl=kwargs.get('verify_ssl', None))
         try:
             tables = p.parse_tables()
         except Exception as caught:
@@ -755,7 +768,8 @@ def read_html(io, match='.+', flavor=None, header=None, index_col=None,
               skiprows=None, attrs=None, parse_dates=False,
               tupleize_cols=False, thousands=',', encoding=None,
               decimal='.', converters=None, na_values=None,
-              keep_default_na=True):
+              keep_default_na=True, username=None, password=None,
+              verify_ssl=False):
     r"""Read HTML tables into a ``list`` of ``DataFrame`` objects.
 
     Parameters
@@ -856,7 +870,16 @@ def read_html(io, match='.+', flavor=None, header=None, index_col=None,
 
         .. versionadded:: 0.19.0
 
-    Returns
+    username : str, default None
+        username for HTTP(s) basic auth
+
+    password : str, default None
+        password for HTTP(s) basic auth
+
+    verify_ssl : bool, default True
+        If False, ssl certificate is not verified (allow self signed SSL certs)
+
+        Returns
     -------
     dfs : list of DataFrames
 
@@ -903,4 +926,5 @@ def read_html(io, match='.+', flavor=None, header=None, index_col=None,
                   parse_dates=parse_dates, tupleize_cols=tupleize_cols,
                   thousands=thousands, attrs=attrs, encoding=encoding,
                   decimal=decimal, converters=converters, na_values=na_values,
-                  keep_default_na=keep_default_na)
+                  keep_default_na=keep_default_na, username=username,
+                  password=password, verify_ssl=verify_ssl)

pandas/io/json/json.py

@@ -174,7 +174,7 @@ def write(self):
 def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
               convert_axes=True, convert_dates=True, keep_default_dates=True,
               numpy=False, precise_float=False, date_unit=None, encoding=None,
-              lines=False):
+              lines=False, username=None, password=None, verify_ssl=None):
     """
     Convert a JSON string to pandas object
 
@@ -263,6 +263,11 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
 
         .. versionadded:: 0.19.0
 
+    username: str, default None. Authentication username for HTTP(s) basic auth
+    passowrd: str, default None. Authentication password for HTTP(s) basic auth
+    verify_ssl: boolean, default None (True).
+                    If false, allow self siged SSL certificates
+
     Returns
     -------
     result : Series or DataFrame, depending on the value of `typ`.
@@ -321,7 +326,10 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
     """
 
     filepath_or_buffer, _, _ = get_filepath_or_buffer(path_or_buf,
-                                                      encoding=encoding)
+                                                      encoding=encoding,
+                                                      username=username,
+                                                      password=password,
+                                                      verify_ssl=verify_ssl)
     if isinstance(filepath_or_buffer, compat.string_types):
         try:
             exists = os.path.exists(filepath_or_buffer)

pandas/io/parsers.py

@@ -391,9 +391,13 @@ def _read(filepath_or_buffer, kwds):
         kwds['encoding'] = encoding
 
     compression = kwds.get('compression')
+    username = kwds.get('username', None)
+    password = kwds.get('password', None)
+    verify_ssl = kwds.get('verify_ssl', None)
     compression = _infer_compression(filepath_or_buffer, compression)
     filepath_or_buffer, _, compression = get_filepath_or_buffer(
-        filepath_or_buffer, encoding, compression)
+        filepath_or_buffer, encoding, compression, username, password,
+        verify_ssl)
     kwds['compression'] = compression
 
     if kwds.get('date_parser', None) is not None:
@@ -574,7 +578,14 @@ def parser_f(filepath_or_buffer,
                  low_memory=_c_parser_defaults['low_memory'],
                  buffer_lines=None,
                  memory_map=False,
-                 float_precision=None):
+                 float_precision=None,
+
+                 # Basic auth (http/https)
+                 username=None,
+                 password=None,
+
+                 # skip verify self signed SSL certificates
+                 verify_ssl=None):
 
         # Alias sep -> delimiter.
         if delimiter is None:
@@ -654,7 +665,12 @@ def parser_f(filepath_or_buffer,
                     mangle_dupe_cols=mangle_dupe_cols,
                     tupleize_cols=tupleize_cols,
                     infer_datetime_format=infer_datetime_format,
-                    skip_blank_lines=skip_blank_lines)
+                    skip_blank_lines=skip_blank_lines,
+
+                    username=username,
+                    password=password,
+                    verify_ssl=verify_ssl
+                    )
 
         return _read(filepath_or_buffer, kwds)
 

pandas/tests/io/test_common.py

@@ -190,6 +190,17 @@ def test_write_fspath_hdf5(self):
 
         tm.assert_frame_equal(result, expected)
 
+    def test_split_url_extract_uname_pwd(self):
+        for url, uname, pwd, nurl in [('https://aaa:[email protected]:1010/aaa.txt',
+                                       'aaa',
+                                       'bbb',
+                                       'https://ccc.com:1010/aaa.txt'
+                                       )]:
+            un, p, u = common.split_uname_from_url(url)
+            assert u == nurl
+            assert un == uname
+            assert p == pwd
+
 
 class TestMMapWrapper(object):