PKG: Exclude data test files.

MANIFEST.in

@@ -10,20 +10,33 @@ prune doc/build
 
 graft pandas
 
-global-exclude *.so
-global-exclude *.pyd
+global-exclude *.bz2
+global-exclude *.csv
+global-exclude *.dta
+global-exclude *.gz
+global-exclude *.h5
+global-exclude *.html
+global-exclude *.json
+global-exclude *.msgpack
+global-exclude *.pickle
+global-exclude *.png
 global-exclude *.pyc
+global-exclude *.pyd
+global-exclude *.sas7bdat
+global-exclude *.so
+global-exclude *.xls
+global-exclude *.xlsm
+global-exclude *.xlsx
+global-exclude *.xpt
+global-exclude *.xz
+global-exclude *.zip
 global-exclude *~
-global-exclude \#*
-global-exclude .git*
 global-exclude .DS_Store
-global-exclude *.png
+global-exclude .git*
+global-exclude \#*
+
+recursive-exclude pandas/tests/io/data
 
-# include examples/data/*
-# recursive-include examples *.py
-# recursive-include doc/source *
-# recursive-include doc/sphinxext *
-# recursive-include LICENSES *
 include versioneer.py
 include pandas/_version.py
 include pandas/io/formats/templates/*.tpl

pandas/tests/io/conftest.py

@@ -9,19 +9,30 @@
 @pytest.fixture(scope='module')
 def tips_file():
     """Path to the tips dataset"""
-    return os.path.join(HERE, 'parser', 'data', 'tips.csv')
+    path = os.path.join(HERE, 'parser', 'data', 'tips.csv')
+    if not os.path.exists(path):
+        pytest.skip("Data files not included in pandas distribution.")
+
+    return path
 
 
 @pytest.fixture(scope='module')
 def jsonl_file():
     """Path a JSONL dataset"""
-    return os.path.join(HERE, 'parser', 'data', 'items.jsonl')
+    path = os.path.join(HERE, 'parser', 'data', 'items.jsonl')
+    if not os.path.exists(path):
+        pytest.skip("Data files not included in pandas distribution.")
+
+    return path
 
 
 @pytest.fixture(scope='module')
 def salaries_table():
     """DataFrame with the salaries dataset"""
     path = os.path.join(HERE, 'parser', 'data', 'salaries.csv')
+    if not os.path.exists(path):
+        pytest.skip("Data files not included in pandas distribution.")
+
     return read_table(path)
 
 
@@ -53,6 +64,9 @@ def s3_resource(tips_file, jsonl_file):
 
     def add_tips_files(bucket_name):
         for s3_key, file_name in test_s3_files:
+            if not os.path.exists(file_name):
+                pytest.skip("Data files not included in pandas distribution.")
+
             with open(file_name, 'rb') as f:
                 conn.Bucket(bucket_name).put_object(
                     Key=s3_key,

pandas/tests/io/test_common.py

@@ -166,6 +166,8 @@ def test_read_non_existant(self, reader, module, error_class, fn_ext):
     ])
     def test_read_fspath_all(self, reader, module, path):
         pytest.importorskip(module)
+        if not os.path.exists(path):
+            pytest.skip("Data files not included in pandas distribution.")
 
         mypath = CustomFSPath(path)
         result = reader(mypath)

pandas/tests/io/test_html.py

@@ -65,9 +65,6 @@ def _skip_if_none_of(module_names):
                 pytest.skip("Bad version of bs4: 4.2.0")
 
 
-DATA_PATH = tm.get_data_path()
-
-
 def assert_framelist_equal(list1, list2, *args, **kwargs):
     assert len(list1) == len(list2), ('lists are not of equal size '
                                       'len(list1) == {0}, '
@@ -86,8 +83,8 @@ def test_bs4_version_fails():
     _skip_if_none_of(('bs4', 'html5lib'))
     import bs4
     if LooseVersion(bs4.__version__) == LooseVersion('4.2.0'):
-        tm.assert_raises(AssertionError, read_html, os.path.join(DATA_PATH,
-                                                                 "spam.html"),
+        tm.assert_raises(AssertionError, read_html,
+                         os.path.join(tm.get_data_path(), "spam.html"),
                          flavor='bs4')
 
 
@@ -100,16 +97,17 @@ def read_html(self, *args, **kwargs):
 
 class TestReadHtml(ReadHtmlMixin):
     flavor = 'bs4'
-    spam_data = os.path.join(DATA_PATH, 'spam.html')
-    spam_data_kwargs = {}
-    if PY3:
-        spam_data_kwargs['encoding'] = 'UTF-8'
-    banklist_data = os.path.join(DATA_PATH, 'banklist.html')
 
     @classmethod
     def setup_class(cls):
         _skip_if_none_of(('bs4', 'html5lib'))
 
+        cls.spam_data = os.path.join(tm.get_data_path(), 'spam.html')
+        cls.spam_data_kwargs = {}
+        if PY3:
+            cls.spam_data_kwargs['encoding'] = 'UTF-8'
+        cls.banklist_data = os.path.join(tm.get_data_path(), 'banklist.html')
+
     def test_to_html_compat(self):
         df = mkdf(4, 3, data_gen_f=lambda *args: rand(), c_idx_names=False,
                   r_idx_names=False).applymap('{0:.3f}'.format).astype(float)
@@ -382,7 +380,7 @@ def test_python_docs_table(self):
     @pytest.mark.slow
     def test_thousands_macau_stats(self):
         all_non_nan_table_index = -2
-        macau_data = os.path.join(DATA_PATH, 'macau.html')
+        macau_data = os.path.join(tm.get_data_path(), 'macau.html')
         dfs = self.read_html(macau_data, index_col=0,
                              attrs={'class': 'style1'})
         df = dfs[all_non_nan_table_index]
@@ -392,7 +390,7 @@ def test_thousands_macau_stats(self):
     @pytest.mark.slow
     def test_thousands_macau_index_col(self):
         all_non_nan_table_index = -2
-        macau_data = os.path.join(DATA_PATH, 'macau.html')
+        macau_data = os.path.join(tm.get_data_path(), 'macau.html')
         dfs = self.read_html(macau_data, index_col=0, header=0)
         df = dfs[all_non_nan_table_index]
 
@@ -520,7 +518,7 @@ def test_countries_municipalities(self):
         assert_framelist_equal(res1, res2)
 
     def test_nyse_wsj_commas_table(self):
-        data = os.path.join(DATA_PATH, 'nyse_wsj.html')
+        data = os.path.join(tm.get_data_path(), 'nyse_wsj.html')
         df = self.read_html(data, index_col=0, header=0,
                             attrs={'class': 'mdcTable'})[0]
 
@@ -542,7 +540,8 @@ def try_remove_ws(x):
 
         df = self.read_html(self.banklist_data, 'Metcalf',
                             attrs={'id': 'table'})[0]
-        ground_truth = read_csv(os.path.join(DATA_PATH, 'banklist.csv'),
+        ground_truth = read_csv(os.path.join(tm.get_data_path(),
+                                             'banklist.csv'),
                                 converters={'Updated Date': Timestamp,
                                             'Closing Date': Timestamp})
         assert df.shape == ground_truth.shape
@@ -660,15 +659,15 @@ def test_parse_dates_combine(self):
         tm.assert_frame_equal(newdf, res[0])
 
     def test_computer_sales_page(self):
-        data = os.path.join(DATA_PATH, 'computer_sales_page.html')
+        data = os.path.join(tm.get_data_path(), 'computer_sales_page.html')
         with tm.assert_raises_regex(ParserError,
                                     r"Passed header=\[0,1\] are "
                                     r"too many rows for this "
                                     r"multi_index of columns"):
             self.read_html(data, header=[0, 1])
 
     def test_wikipedia_states_table(self):
-        data = os.path.join(DATA_PATH, 'wikipedia_states.html')
+        data = os.path.join(tm.get_data_path(), 'wikipedia_states.html')
         assert os.path.isfile(data), '%r is not a file' % data
         assert os.path.getsize(data), '%r is an empty file' % data
         result = self.read_html(data, 'Arizona', header=1)[0]
@@ -788,11 +787,14 @@ def _lang_enc(filename):
 
 
 class TestReadHtmlEncoding(object):
-    files = glob.glob(os.path.join(DATA_PATH, 'html_encoding', '*.html'))
     flavor = 'bs4'
 
     @classmethod
     def setup_class(cls):
+        cls.files = glob.glob(os.path.join(tm.get_data_path(),
+                                           'html_encoding',
+                                           '*.html'))
+
         _skip_if_none_of((cls.flavor, 'html5lib'))
 
     def read_html(self, *args, **kwargs):
@@ -847,8 +849,8 @@ def setup_class(cls):
 
     def test_data_fail(self):
         from lxml.etree import XMLSyntaxError
-        spam_data = os.path.join(DATA_PATH, 'spam.html')
-        banklist_data = os.path.join(DATA_PATH, 'banklist.html')
+        spam_data = os.path.join(tm.get_data_path(), 'spam.html')
+        banklist_data = os.path.join(tm.get_data_path(), 'banklist.html')
 
         with pytest.raises(XMLSyntaxError):
             self.read_html(spam_data)
@@ -857,15 +859,15 @@ def test_data_fail(self):
             self.read_html(banklist_data)
 
     def test_works_on_valid_markup(self):
-        filename = os.path.join(DATA_PATH, 'valid_markup.html')
+        filename = os.path.join(tm.get_data_path(), 'valid_markup.html')
         dfs = self.read_html(filename, index_col=0)
         assert isinstance(dfs, list)
         assert isinstance(dfs[0], DataFrame)
 
     @pytest.mark.slow
     def test_fallback_success(self):
         _skip_if_none_of(('bs4', 'html5lib'))
-        banklist_data = os.path.join(DATA_PATH, 'banklist.html')
+        banklist_data = os.path.join(tm.get_data_path(), 'banklist.html')
         self.read_html(banklist_data, '.*Water.*', flavor=['lxml', 'html5lib'])
 
     def test_to_html_timestamp(self):
@@ -893,7 +895,7 @@ def test_parse_dates_combine(self):
         tm.assert_frame_equal(newdf, res[0])
 
     def test_computer_sales_page(self):
-        data = os.path.join(DATA_PATH, 'computer_sales_page.html')
+        data = os.path.join(tm.get_data_path(), 'computer_sales_page.html')
         self.read_html(data, header=[0, 1])
 
 
@@ -914,7 +916,7 @@ def get_elements_from_file(url, element='table'):
 
 @pytest.mark.slow
 def test_bs4_finds_tables():
-    filepath = os.path.join(DATA_PATH, "spam.html")
+    filepath = os.path.join(tm.get_data_path(), "spam.html")
     with warnings.catch_warnings():
         warnings.filterwarnings('ignore')
         assert get_elements_from_file(filepath, 'table')
@@ -929,19 +931,19 @@ def get_lxml_elements(url, element):
 
 @pytest.mark.slow
 def test_lxml_finds_tables():
-    filepath = os.path.join(DATA_PATH, "spam.html")
+    filepath = os.path.join(tm.get_data_path(), "spam.html")
     assert get_lxml_elements(filepath, 'table')
 
 
 @pytest.mark.slow
 def test_lxml_finds_tbody():
-    filepath = os.path.join(DATA_PATH, "spam.html")
+    filepath = os.path.join(tm.get_data_path(), "spam.html")
     assert get_lxml_elements(filepath, 'tbody')
 
 
 def test_same_ordering():
     _skip_if_none_of(['bs4', 'lxml', 'html5lib'])
-    filename = os.path.join(DATA_PATH, 'valid_markup.html')
+    filename = os.path.join(tm.get_data_path(), 'valid_markup.html')
     dfs_lxml = read_html(filename, index_col=0, flavor=['lxml'])
     dfs_bs4 = read_html(filename, index_col=0, flavor=['bs4'])
     assert_framelist_equal(dfs_lxml, dfs_bs4)
@@ -965,7 +967,7 @@ def test_importcheck_thread_safety():
     pytest.importorskip('lxml')
     reload(pandas.io.html)
 
-    filename = os.path.join(DATA_PATH, 'valid_markup.html')
+    filename = os.path.join(tm.get_data_path(), 'valid_markup.html')
     helper_thread1 = ErrorThread(target=read_html, args=(filename,))
     helper_thread2 = ErrorThread(target=read_html, args=(filename,))
 

pandas/tests/io/test_packers.py

@@ -830,6 +830,8 @@ def test_default_encoding(self):
 def legacy_packers_versions():
     # yield the packers versions
     path = tm.get_data_path('legacy_msgpack')
+    if not os.path.exists(path):
+        raise pytest.skip("Data file {} does not exist.".format(path))
     for v in os.listdir(path):
         p = os.path.join(path, v)
         if os.path.isdir(p):

pandas/tests/io/test_pickle.py

@@ -191,6 +191,8 @@ def compare_sp_frame_float(result, expected, typ, version):
 def legacy_pickle_versions():
     # yield the pickle versions
     path = tm.get_data_path('legacy_pickle')
+    if not os.path.exists(path):
+        raise pytest.skip("Data path {} does not exists.".format(path))
     for v in os.listdir(path):
         p = os.path.join(path, v)
         if os.path.isdir(p):

pandas/tests/plotting/common.py

@@ -77,6 +77,10 @@ def setup_method(self, method):
         from pandas import read_csv
         base = os.path.join(os.path.dirname(curpath()), os.pardir)
         path = os.path.join(base, 'tests', 'data', 'iris.csv')
+
+        if not os.path.exists(path):
+            pytest.skip("Data files not included in pandas distribution.")
+
         self.iris = read_csv(path)
 
         n = 100

pandas/util/_test_decorators.py

@@ -23,6 +23,8 @@ def test_foo():
 
 For more information, refer to the ``pytest`` documentation on ``skipif``.
 """
+import functools
+import os
 
 import pytest
 import locale

pandas/util/testing.py

@@ -732,10 +732,16 @@ def get_data_path(f=''):
     """Return the path of a data file, these are relative to the current test
     directory.
     """
+    import pytest
+
     # get our callers file
     _, filename, _, _, _, _ = inspect.getouterframes(inspect.currentframe())[1]
     base_dir = os.path.abspath(os.path.dirname(filename))
-    return os.path.join(base_dir, 'data', f)
+    path = os.path.join(base_dir, 'data', f)
+    if not os.path.exists(path):
+        pytest.skip("Data files not included in pandas distribution.")
+
+    return path
 
 # -----------------------------------------------------------------------------
 # Comparators

setup.py

@@ -722,11 +722,7 @@ def pxd(name):
       maintainer=AUTHOR,
       version=versioneer.get_version(),
       packages=find_packages(include=['pandas', 'pandas.*']),
-      package_data={'': ['data/*', 'templates/*'],
-                    'pandas.tests.io': ['data/legacy_hdf/*.h5',
-                                        'data/legacy_pickle/*/*.pickle',
-                                        'data/legacy_msgpack/*/*.msgpack',
-                                        'data/html_encoding/*.html']},
+      package_data={'': ['templates/*']},
       ext_modules=extensions,
       maintainer_email=EMAIL,
       description=DESCRIPTION,