Merge pull request #145 from jtkiley/master

Adds retrieval of EDGAR master index.

Author: davidastephens

docs/source/remote_data.rst

@@ -396,3 +396,16 @@ You can specify dataset ID "tran_sf_railac" to get corresponding data via ``Data
 
     df = web.DataReader("tran_sf_railac", 'eurostat')
     df
+
+.. _remote_data.edgar:
+
+EDGAR Index
+===========
+
+Company filing index from EDGAR (SEC).
+
+.. ipython:: python
+
+    import pandas_datareader.data as web
+    ed = web.DataReader('full', 'edgar-index')
+    ed[:5]

docs/source/whatsnew/v0.2.2.txt

@@ -19,6 +19,7 @@ New features
 ~~~~~~~~~~~~
 
 - ``DataReader`` now supports dividend only pulls from Yahoo! Finance, see :ref:`here<remote_data.yahoo>` (:issue:`138`).
+- ``DataReader`` now supports SEC EDGAR full (current quarter) index retrieval, see :ref:`here<remote_data.edgar>` (:issue:`143`).
 
 .. _whatsnew_022.api_breaking:
 

pandas_datareader/base.py

@@ -16,6 +16,9 @@
 
 from pandas_datareader._utils import RemoteDataError, SymbolWarning
 
+import requests_ftp
+requests_ftp.monkeypatch_session()
+
 
 class _BaseReader(object):
 

pandas_datareader/data.py

@@ -17,6 +17,7 @@
 from pandas_datareader.fred import FredReader
 from pandas_datareader.famafrench import FamaFrenchReader
 from pandas_datareader.oecd import OECDReader
+from pandas_datareader.edgar import EdgarIndexReader
 
 
 def get_data_fred(*args, **kwargs):
@@ -43,8 +44,8 @@ def DataReader(name, data_source=None, start=None, end=None,
     """
     Imports data from a number of online sources.
 
-    Currently supports Yahoo! Finance, Google Finance, St. Louis FED (FRED)
-    and Kenneth French's data library.
+    Currently supports Yahoo! Finance, Google Finance, St. Louis FED (FRED),
+    Kenneth French's data library, and the SEC's EDGAR Index.
 
     Parameters
     ----------
@@ -53,7 +54,7 @@ def DataReader(name, data_source=None, start=None, end=None,
         accept a list of names.
     data_source: {str, None}
         the data source ("yahoo", "yahoo-actions", "yahoo-dividends",
-        "google", "fred", or "ff")
+        "google", "fred", "ff", or "edgar-index")
     start : {datetime, None}
         left boundary for range (defaults to 1/1/2010)
     end : {datetime, None}
@@ -86,6 +87,9 @@ def DataReader(name, data_source=None, start=None, end=None,
     ff = DataReader("F-F_Research_Data_Factors_weekly", "famafrench")
     ff = DataReader("6_Portfolios_2x3", "famafrench")
     ff = DataReader("F-F_ST_Reversal_Factor", "famafrench")
+
+    # Data from EDGAR index
+    ed = DataReader("master", "edgar-index")
     """
     if data_source == "yahoo":
         return YahooDailyReader(symbols=name, start=start, end=end,
@@ -127,6 +131,10 @@ def DataReader(name, data_source=None, start=None, end=None,
         return EurostatReader(symbols=name, start=start, end=end,
                               retry_count=retry_count, pause=pause,
                               session=session).read()
+    elif data_source == "edgar-index":
+        return EdgarIndexReader(symbols=name, start=start, end=end,
+                                retry_count=retry_count, pause=pause,
+                                session=session).read()
     else:
         raise NotImplementedError(
                 "data_source=%r is not implemented" % data_source)

pandas_datareader/edgar.py

@@ -0,0 +1,43 @@
+from pandas import read_csv
+from pandas.io.common import ZipFile
+from pandas.compat import StringIO
+from pandas.compat import BytesIO
+
+from pandas_datareader.base import _BaseReader
+
+
+_URL_FULL = 'ftp://ftp.sec.gov/edgar/full-index/master.zip'
+_COLUMNS = ['cik', 'company_name', 'form_type', 'date_filed', 'filename']
+
+
+class EdgarIndexReader(_BaseReader):
+    """
+    Get master index from the SEC's EDGAR database.
+
+    Returns
+    -------
+    edgar_index : pandas.DataFrame.
+        DataFrame of EDGAR master index.
+    """
+
+    @property
+    def url(self):
+        return _URL_FULL
+
+    def _read_zipfile(self, url):
+
+        zipf = BytesIO(self._get_response(url).content)
+
+        with ZipFile(zipf, 'r') as zf:
+            data = zf.open(zf.namelist()[0]).read().decode()
+
+        return data
+
+    def _read_one_data(self, url, params):
+
+        index_file = StringIO(self._read_zipfile(url))
+
+        index = read_csv(index_file, delimiter='|', header=None,
+                         index_col=False, skiprows=10, names=_COLUMNS,
+                         low_memory=False)
+        return index

pandas_datareader/tests/test_data.py

@@ -22,7 +22,7 @@
 import pandas_datareader.data as web
 from pandas_datareader.data import (DataReader, GoogleDailyReader, YahooDailyReader,
                                     YahooQuotesReader, YahooActionReader,
-                                    FredReader, OECDReader)
+                                    FredReader, OECDReader, EdgarIndexReader)
 from pandas_datareader._utils import SymbolWarning, RemoteDataError
 from pandas_datareader.yahoo.quotes import _yahoo_codes
 
@@ -530,6 +530,10 @@ def test_read_fred(self):
         vix = DataReader("VIXCLS", "fred")
         assert isinstance(vix, DataFrame)
 
+    def test_read_edgar_index(self):
+        ed = DataReader("full", "edgar-index")
+        assert isinstance(ed, DataFrame)
+
     def test_not_implemented(self):
         self.assertRaises(NotImplementedError, DataReader, "NA", "NA")
 

pandas_datareader/tests/test_edgar.py

@@ -0,0 +1,14 @@
+import nose
+import pandas.util.testing as tm
+
+import pandas_datareader.data as web
+
+
+class TestEdgarIndex(tm.TestCase):
+    def test_get_index(self):
+        ed = web.DataReader('full', 'edgar-index')
+        assert len(ed > 1000)
+
+if __name__ == '__main__':
+    nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
+                   exit=False)

setup.py

@@ -24,7 +24,7 @@ def readme():
         return f.read()
 
 INSTALL_REQUIRES = (
-    ['pandas', 'requests', 'requests-file']
+    ['pandas', 'requests', 'requests-file', 'requests-ftp']
 )
 
 setup(