Replace Yahoo iCharts API (#355)

* Replaces ichart API for single-stock price exports from Yahoo, multi-stock still failing (#315)

Restores change necessary for Google to function
Fixes yahoo-actions per API endpoint update
Update regex pattern for crumbs, per heyuhere's review
'v' is no longer a valid interval value
Fixes Yahoo intervals and cases where the Yahoo cookie could not be extracted.
Implements multi-stock queries to Yahoo API
Adds a pause multiplier for subsequent requests from Yahoo, error handling for empty data requests, and updates some test logic for pandas 0.20.x (notably ix deprecation)
Check object type before checking contents
Replacement regex logic for additional Yahoo cookie token structures, per chris-b1
Improved error handling and refactoring test to best practices, per jreback review.

closes #315

* better error handling after get_response

* docs for 0.5.0

* remove deprecation warnings: ix usage -> loc/iloc
remove deprecation warnings: sortlevel usage -> sort_index

* more resource cleaning

* update changelog

* skip enigma tests locally if no api key

* fixturize test_yahoo_options

* add in test.sh script

* CI: use trusty dist

Author: jreback

.travis.yml

@@ -5,33 +5,30 @@ language: python
 matrix:
   fast_finish: true
   include:
-  - os: linux
+  - dist: trusty
     env:
       - PYTHON=2.7 PANDAS=0.17.1
-  - os: linux
+  - dist: trusty
     env:
       - PYTHON=2.7 PANDAS=0.19.2
-  - os: linux
+  - dist: trusty
     env:
       - PYTHON=3.5 PANDAS=0.17.1
-  - os: linux
+  - dist: trusty
     env:
       - PYTHON=3.5 PANDAS=0.18.1
-  - os: linux
-    env:
-      - PYTHON=3.5 PANDAS=0.19.2
-  - os: linux
+  - dist: trusty
     env:
       - PYTHON=3.6 PANDAS=0.19.2
-  - os: linux
+  - dist: trusty
     env:
-      - PYTHON=3.6 PANDAS=0.20.1
+      - PYTHON=3.6 PANDAS=0.20.2
   # In allow failures
-  - os: linux
+  - dist: trusty
     env:
       - PYTHON=3.6 PANDAS="MASTER"
   allow_failures:
-    - os: linux
+    - dist: trusty
       env:
         - PYTHON=3.6 PANDAS="MASTER"
 
@@ -70,7 +67,7 @@ install:
 
 script:
     - export ENIGMA_API_KEY=$ENIGMA_API_KEY
-    - pytest -s --cov=pandas_datareader --cov-report xml:/tmp/cov-datareader.xml --junitxml=/tmp/datareader.xml
+    - pytest -s -r xX --cov=pandas_datareader --cov-report xml:/tmp/cov-datareader.xml --junitxml=/tmp/datareader.xml
     - flake8 --version
     - flake8 pandas_datareader
 

docs/source/whatsnew.rst

@@ -18,6 +18,7 @@ What's New
 
 These are new features and improvements of note in each release.
 
+.. include:: whatsnew/v0.5.0.txt
 .. include:: whatsnew/v0.4.0.txt
 .. include:: whatsnew/v0.3.0.txt
 .. include:: whatsnew/v0.2.1.txt

docs/source/whatsnew/v0.5.0.txt

@@ -0,0 +1,29 @@
+.. _whatsnew_050:
+
+v0.5.0 (July ??, 2017)
+----------------------
+
+This is a major release from 0.4.0.
+
+Highlights include:
+
+.. contents:: What's new in v0.5.0
+    :local:
+    :backlinks: none
+
+.. _whatsnew_050.enhancements:
+
+Enhancements
+~~~~~~~~~~~~
+
+- Compat with new Yahoo API (:issue:`315`)
+
+.. _whatsnew_050.bug_fixes:
+
+Bug Fixes
+~~~~~~~~~
+
+- web sessions are closed properly at the end of use (:issue:`355`)
+- Handle commas in large price quotes (:issue:`345`)
+- Test suite fixes for test_get_options_data (:issue:`352`)
+- Test suite fixes for test_wdi_download (:issue:`350`)

pandas_datareader/base.py

@@ -53,8 +53,13 @@ def __init__(self, symbols, start=None, end=None,
         self.retry_count = retry_count
         self.pause = pause
         self.timeout = timeout
+        self.pause_multiplier = 1
         self.session = _init_session(session, retry_count)
 
+    def close(self):
+        """ close my session """
+        self.session.close()
+
     @property
     def url(self):
         # must be overridden in subclass
@@ -66,7 +71,10 @@ def params(self):
 
     def read(self):
         """ read data """
-        return self._read_one_data(self.url, self.params)
+        try:
+            return self._read_one_data(self.url, self.params)
+        finally:
+            self.close()
 
     def _read_one_data(self, url, params):
         """ read one data from specified URL """
@@ -85,6 +93,10 @@ def _read_url_as_StringIO(self, url, params=None):
         response = self._get_response(url, params=params)
         text = self._sanitize_response(response)
         out = StringIO()
+        if len(text) == 0:
+            service = self.__class__.__name__
+            raise IOError("{} request returned no data; check URL for invalid "
+                          "inputs: {}".format(service, self.url))
         if isinstance(text, compat.binary_type):
             out.write(bytes_to_str(text))
         else:
@@ -99,7 +111,7 @@ def _sanitize_response(response):
         """
         return response.content
 
-    def _get_response(self, url, params=None):
+    def _get_response(self, url, params=None, headers=None):
         """ send raw HTTP request to get requests.Response from the specified url
         Parameters
         ----------
@@ -110,15 +122,29 @@ def _get_response(self, url, params=None):
         """
 
         # initial attempt + retry
+        pause = self.pause
         for i in range(self.retry_count + 1):
-            response = self.session.get(url, params=params)
+            response = self.session.get(url,
+                                        params=params,
+                                        headers=headers)
             if response.status_code == requests.codes.ok:
                 return response
-            time.sleep(self.pause)
+
+            time.sleep(pause)
+
+            # Increase time between subsequent requests, per subclass.
+            pause *= self.pause_multiplier
+            # Get a new breadcrumb if necessary, in case ours is invalidated
+            if isinstance(params, list) and 'crumb' in params:
+                params['crumb'] = self._get_crumb(self.retry_count)
         if params is not None and len(params) > 0:
             url = url + "?" + urlencode(params)
         raise RemoteDataError('Unable to read URL: {0}'.format(url))
 
+    def _get_crumb(self, *args):
+        """ To be implemented by subclass """
+        raise NotImplementedError("Subclass has not implemented method.")
+
     def _read_lines(self, out):
         rs = read_csv(out, index_col=0, parse_dates=True, na_values='-')[::-1]
         # Yahoo! Finance sometimes does this awesome thing where they

pandas_datareader/data.py

@@ -9,7 +9,7 @@
 
 from pandas_datareader.yahoo.daily import YahooDailyReader
 from pandas_datareader.yahoo.quotes import YahooQuotesReader
-from pandas_datareader.yahoo.actions import YahooActionReader
+from pandas_datareader.yahoo.actions import (YahooActionReader, YahooDivReader)
 from pandas_datareader.yahoo.components import _get_data as get_components_yahoo  # noqa
 from pandas_datareader.yahoo.options import Options as YahooOptions
 from pandas_datareader.google.options import Options as GoogleOptions
@@ -121,10 +121,10 @@ def DataReader(name, data_source=None, start=None, end=None,
                                  retry_count=retry_count, pause=pause,
                                  session=session).read()
     elif data_source == "yahoo-dividends":
-        return YahooDailyReader(symbols=name, start=start, end=end,
-                                adjust_price=False, chunksize=25,
-                                retry_count=retry_count, pause=pause,
-                                session=session, interval='v').read()
+        return YahooDivReader(symbols=name, start=start, end=end,
+                              adjust_price=False, chunksize=25,
+                              retry_count=retry_count, pause=pause,
+                              session=session, interval='d').read()
 
     elif data_source == "google":
         return GoogleDailyReader(symbols=name, start=start, end=end,

pandas_datareader/edgar.py

@@ -150,6 +150,12 @@ def _fix_old_file_paths(self, path):
         return path
 
     def read(self):
+        try:
+            return self._read()
+        finally:
+            self.close()
+
+    def _read(self):
         try:
             self._sec_ftp_session = FTP(_SEC_FTP, timeout=self.timeout)
             self._sec_ftp_session.login()

pandas_datareader/enigma.py

@@ -100,6 +100,12 @@ def extract_export_url(self, delay=10, max_attempts=10):
             return resp.json()[self.export_key]
 
     def read(self):
+        try:
+            return self._read()
+        finally:
+            self.close()
+
+    def _read(self):
         export_gzipped_req = self._request(self.extract_export_url())
         decompressed_data = self._decompress_export(
             export_gzipped_req.content).decode("utf-8")

pandas_datareader/fred.py

@@ -20,6 +20,12 @@ def url(self):
         return "http://research.stlouisfed.org/fred2/series/"
 
     def read(self):
+        try:
+            return self._read()
+        finally:
+            self.close()
+
+    def _read(self):
         if not is_list_like(self.symbols):
             names = [self.symbols]
         else:

pandas_datareader/tests/google/test_google.py

@@ -83,21 +83,21 @@ def assert_option_result(self, df):
 
     def test_get_quote_string(self):
         df = web.get_quote_google('GOOG')
-        assert df.ix['GOOG']['last'] > 0.0
+        assert df.loc['GOOG', 'last'] > 0.0
         tm.assert_index_equal(df.index, pd.Index(['GOOG']))
         self.assert_option_result(df)
 
     def test_get_quote_stringlist(self):
         df = web.get_quote_google(['GOOG', 'AMZN', 'GOOG'])
-        assert_series_equal(df.ix[0], df.ix[2])
+        assert_series_equal(df.iloc[0], df.iloc[2])
         tm.assert_index_equal(df.index, pd.Index(['GOOG', 'AMZN', 'GOOG']))
         self.assert_option_result(df)
 
     def test_get_goog_volume(self):
         for locale in self.locales:
             with tm.set_locale(locale):
                 df = web.get_data_google('GOOG').sort_index()
-            assert df.Volume.ix['JAN-02-2015'] == 1446662
+            assert df.Volume.loc['JAN-02-2015'] == 1446662
 
     def test_get_multi1(self):
         for locale in self.locales:
@@ -130,13 +130,13 @@ def test_get_multi2(self):
                 with tm.set_locale(locale):
                     pan = web.get_data_google(['GE', 'MSFT', 'INTC'],
                                               'JAN-01-12', 'JAN-31-12')
-                result = pan.Close.ix['01-18-12']
+                result = pan.Close.loc['01-18-12']
                 assert_n_failed_equals_n_null_columns(w, result)
 
                 # sanity checking
 
                 assert np.issubdtype(result.dtype, np.floating)
-                result = pan.Open.ix['Jan-15-12':'Jan-20-12']
+                result = pan.Open.loc['Jan-15-12':'Jan-20-12']
 
                 assert result.shape == (4, 3)
                 assert_n_failed_equals_n_null_columns(w, result)
@@ -158,7 +158,7 @@ def test_unicode_date(self):
     def test_google_reader_class(self):
         r = GoogleDailyReader('GOOG')
         df = r.read()
-        assert df.Volume.ix['JAN-02-2015'] == 1446662
+        assert df.Volume.loc['JAN-02-2015'] == 1446662
 
         session = requests.Session()
         r = GoogleDailyReader('GOOG', session=session)

pandas_datareader/tests/google/test_options.py

@@ -7,8 +7,6 @@
 import pandas.util.testing as tm
 
 import pandas_datareader.data as web
-from pandas_datareader._utils import RemoteDataError
-from pandas_datareader._testing import skip_on_exception
 
 
 class TestGoogleOptions(object):
@@ -18,7 +16,6 @@ def setup_class(cls):
         # GOOG has monthlies
         cls.goog = web.Options('GOOG', 'google')
 
-    @skip_on_exception(RemoteDataError)
     def test_get_options_data(self):
         options = self.goog.get_options_data(expiry=self.goog.expiry_dates[0])
 
@@ -46,7 +43,6 @@ def test_get_options_data_yearmonth(self):
         with pytest.raises(NotImplementedError):
             self.goog.get_options_data(month=1, year=2016)
 
-    @skip_on_exception(RemoteDataError)
     def test_expiry_dates(self):
         dates = self.goog.expiry_dates
 

pandas_datareader/tests/io/test_jsdmx.py

@@ -50,7 +50,7 @@ def test_land_use(self):
         result = read_jsdmx(os.path.join(self.dirpath, 'jsdmx',
                                          'land_use.json'))
         assert isinstance(result, pd.DataFrame)
-        result = result.ix['2010':'2011']
+        result = result.loc['2010':'2011']
 
         exp_col = pd.MultiIndex.from_product([
             ['Japan', 'United States'],

pandas_datareader/tests/test_data.py

@@ -4,6 +4,8 @@
 import pandas_datareader.data as web
 
 from pandas import DataFrame
+from pandas_datareader._utils import RemoteDataError
+from pandas_datareader._testing import skip_on_exception
 from pandas_datareader.data import DataReader
 
 
@@ -15,10 +17,13 @@ def test_options_source_warning(self):
 
 
 class TestDataReader(object):
+
+    @skip_on_exception(RemoteDataError)
     def test_read_yahoo(self):
         gs = DataReader("GS", "yahoo")
         assert isinstance(gs, DataFrame)
 
+    @pytest.mark.xfail(RemoteDataError, reason="failing after #355")
     def test_read_yahoo_dividends(self):
         gs = DataReader("GS", "yahoo-dividends")
         assert isinstance(gs, DataFrame)

pandas_datareader/tests/test_edgar.py

@@ -4,9 +4,6 @@
 import pandas.util.testing as tm
 import pandas_datareader.data as web
 
-from pandas_datareader._utils import RemoteDataError
-from pandas_datareader._testing import skip_on_exception
-
 
 class TestEdgarIndex(object):
 
@@ -16,7 +13,6 @@ def setup_class(cls):
         # Disabling tests until re-write.
         pytest.skip("Disabling tests until re-write.")
 
-    @skip_on_exception(RemoteDataError)
     def test_get_full_index(self):
         ed = web.DataReader('full', 'edgar-index')
         assert len(ed) > 1000
@@ -25,7 +21,6 @@ def test_get_full_index(self):
                                 'date_filed', 'filename'], dtype='object')
         tm.assert_index_equal(ed.columns, exp_columns)
 
-    @skip_on_exception(RemoteDataError)
     def test_get_nonzip_index_and_low_date(self):
         ed = web.DataReader('daily', 'edgar-index', '1994-06-30',
                             '1994-07-02')
@@ -38,14 +33,12 @@ def test_get_nonzip_index_and_low_date(self):
                                 'filename'], dtype='object')
         tm.assert_index_equal(ed.columns, exp_columns)
 
-    @skip_on_exception(RemoteDataError)
     def test_get_gz_index_and_no_date(self):
         # TODO: Rewrite, as this test causes Travis to timeout.
 
         ed = web.DataReader('daily', 'edgar-index')
         assert len(ed) > 2000
 
-    @skip_on_exception(RemoteDataError)
     def test_6_digit_date(self):
         ed = web.DataReader('daily', 'edgar-index', start='1998-05-18',
                             end='1998-05-18')