From dd857ba9b724bea2f3f88851259045e8250481dc Mon Sep 17 00:00:00 2001 From: David Stephens Date: Fri, 27 Mar 2015 20:06:16 -0700 Subject: [PATCH 1/2] BUG: Bring pandas up to date with pandas_datareader --- pandas/io/data.py | 17 +- pandas/io/tests/data/yahoo_options3.html | 2807 ++++++++++++++++++++++ pandas/io/tests/test_data.py | 7 + 3 files changed, 2821 insertions(+), 10 deletions(-) create mode 100644 pandas/io/tests/data/yahoo_options3.html diff --git a/pandas/io/data.py b/pandas/io/data.py index ea635e85ed177..936e5facd3b79 100644 --- a/pandas/io/data.py +++ b/pandas/io/data.py @@ -172,14 +172,14 @@ def _retry_read_url(url, retry_count, pause, name): if len(rs) > 2 and rs.index[-1] == rs.index[-2]: # pragma: no cover rs = rs[:-1] - #Get rid of unicode characters in index name. - try: - rs.index.name = rs.index.name.decode('unicode_escape').encode('ascii', 'ignore') - except AttributeError: - #Python 3 string has no decode method. - rs.index.name = rs.index.name.encode('ascii', 'ignore').decode() + #Get rid of unicode characters in index name. + try: + rs.index.name = rs.index.name.decode('unicode_escape').encode('ascii', 'ignore') + except AttributeError: + #Python 3 string has no decode method. + rs.index.name = rs.index.name.encode('ascii', 'ignore').decode() - return rs + return rs raise IOError("after %d tries, %s did not " "return a 200 for url %r" % (retry_count, name, url)) @@ -701,9 +701,6 @@ def _option_frames_from_url(self, url): calls = frames[self._TABLE_LOC['calls']] puts = frames[self._TABLE_LOC['puts']] - if len(calls) == 0 or len(puts) == 0: - raise RemoteDataError('Received no data from Yahoo at url: %s' % url) - calls = self._process_data(calls, 'call') puts = self._process_data(puts, 'put') diff --git a/pandas/io/tests/data/yahoo_options3.html b/pandas/io/tests/data/yahoo_options3.html new file mode 100644 index 0000000000000..6e79bb9bf9f36 --- /dev/null +++ b/pandas/io/tests/data/yahoo_options3.html @@ -0,0 +1,2807 @@ + + + + + SPWR Option Chain | Yahoo! Inc. Stock - Yahoo! Finance + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ +
  • FirefoxInstall the new Firefox »
    • Mail
    +
    + + +
    + +
    + +
    + + + + + + + +
    + + +
    + + +
    + +
    + + + +
    +
    +
    +
    + + + + Dow + + + + Down + + + 0.58% + + + + + + + Nasdaq + + + + Down + + + 0.32% + + + + + + +
    + +
    +
    +
    + + +
    + +
    +

    More on SPWR

    +
    +
    + + +

    Quotes

    + + +

    Charts

    + + +

    News & Info

    + + +

    Company

    + + +

    Analyst Coverage

    + + +

    Ownership

    + + +

    Financials

    + + +
    +
    + +
    +
    + +
    +
    +
    +
    +
    + +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + + +
    + + +
    +
    +
    +
    +
    +
    +

    SunPower Corporation (SPWR)

    + +
    +
    +
    +
    + + 33.05 + + + + + Up +0.07(0.21%) + + + NASDAQ - As of 4:00PM EDT + +
    +
    | + + After Hours: + 33.10 Up +0.05 (0.15%) 7:47PM EDT + + +
    +
    + +
    + + +
    +
    +
    +
    +
    + + +
    + + + +
    + + +
    +
    + +
    +
    May 1, 2015
    + +
    + + + +
    +
    +
    + + +
    +
    +
    +
    +
    + + +
    +
    + In The Money +
    +
    + + + +
    +
    +

    Show Me Strikes From

    +
    + $ + to $ +
    + Apply Filter + Clear Filter +
    + + + + + +
    + +
    +
    +

    Show Me Strikes From

    +
    + $ + to $ +
    + Apply Filter + Clear Filter +
    + + + + + +
    + + +
    +
    + + + +
    + + +
    + +
    +
    + + + +
    +
    + + + + + + + + + + + + + + + +
    + + + + + + + + + + + + \ No newline at end of file diff --git a/pandas/io/tests/test_data.py b/pandas/io/tests/test_data.py index 937af834d348b..6b29ffb9d6a7b 100644 --- a/pandas/io/tests/test_data.py +++ b/pandas/io/tests/test_data.py @@ -297,6 +297,7 @@ def setUpClass(cls): cls.dirpath = tm.get_data_path() cls.html1 = os.path.join(cls.dirpath, 'yahoo_options1.html') cls.html2 = os.path.join(cls.dirpath, 'yahoo_options2.html') + cls.html3 = os.path.join(cls.dirpath, 'yahoo_options3.html') #Empty table GH#22 cls.data1 = cls.aapl._option_frames_from_url(cls.html1)['puts'] @classmethod @@ -428,6 +429,12 @@ def test_month_year(self): self.assertTrue(len(data) > 1) + @network + def test_empty_table(self): + #GH22 + empty = self.aapl._option_frames_from_url(self.html3)['puts'] + self.assertTrue(len(empty) == 0) + class TestOptionsWarnings(tm.TestCase): @classmethod From 1b205937809df225bef1e49de217262bc0359ab0 Mon Sep 17 00:00:00 2001 From: David Stephens Date: Sat, 28 Mar 2015 12:14:32 -0700 Subject: [PATCH 2/2] COMPAT: Fix dl_mult_symbols for Python3 --- pandas/io/data.py | 10 +++++++--- pandas/io/tests/test_data.py | 4 ++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pandas/io/data.py b/pandas/io/data.py index 936e5facd3b79..3e077bf526ab9 100644 --- a/pandas/io/data.py +++ b/pandas/io/data.py @@ -326,18 +326,23 @@ def _dl_mult_symbols(symbols, start, end, interval, chunksize, retry_count, paus method): stocks = {} failed = [] + passed = [] for sym_group in _in_chunks(symbols, chunksize): for sym in sym_group: try: stocks[sym] = method(sym, start, end, interval, retry_count, pause) + passed.append(sym) except IOError: warnings.warn('Failed to read symbol: {0!r}, replacing with ' 'NaN.'.format(sym), SymbolWarning) failed.append(sym) + if len(passed) == 0: + raise RemoteDataError("No data fetched using " + "{0!r}".format(method.__name__)) try: - if len(stocks) > 0 and len(failed) > 0: - df_na = stocks.values()[0].copy() + if len(stocks) > 0 and len(failed) > 0 and len(passed) > 0: + df_na = stocks[passed[0]].copy() df_na[:] = np.nan for sym in failed: stocks[sym] = df_na @@ -347,7 +352,6 @@ def _dl_mult_symbols(symbols, start, end, interval, chunksize, retry_count, paus raise RemoteDataError("No data fetched using " "{0!r}".format(method.__name__)) - _source_functions = {'google': _get_hist_google, 'yahoo': _get_hist_yahoo} diff --git a/pandas/io/tests/test_data.py b/pandas/io/tests/test_data.py index 6b29ffb9d6a7b..70a25a45c0ad4 100644 --- a/pandas/io/tests/test_data.py +++ b/pandas/io/tests/test_data.py @@ -101,6 +101,10 @@ def test_get_multi_invalid(self): self.assertIn('INVALID', pan.minor_axis) @network + def test_get_multi_all_invalid(self): + sl = ['INVALID', 'INVALID2', 'INVALID3'] + self.assertRaises(RemoteDataError, web.get_data_google, sl, '2012') + def test_get_multi2(self): with warnings.catch_warnings(record=True) as w: for locale in self.locales: