Merge pull request #5073 from cpcloud/travis-locale

TST/CI: make sure that locales are tested

Author: cpcloud

.travis.yml

@@ -6,13 +6,13 @@ python:
 matrix:
   include:
     - python: 2.6
-      env: NOSE_ARGS="not slow" CLIPBOARD=xclip
+      env: NOSE_ARGS="not slow" CLIPBOARD=xclip LOCALE_OVERRIDE="it_IT.UTF-8"
     - python: 2.7
       env: NOSE_ARGS="slow and not network" LOCALE_OVERRIDE="zh_CN.GB18030" FULL_DEPS=true JOB_TAG=_LOCALE
     - python: 2.7
-      env: NOSE_ARGS="not slow" FULL_DEPS=true GUI=gtk2
+      env: NOSE_ARGS="not slow" FULL_DEPS=true CLIPBOARD_GUI=gtk2
     - python: 3.2
-      env: NOSE_ARGS="not slow" FULL_DEPS=true GUI=qt4
+      env: NOSE_ARGS="not slow" FULL_DEPS=true CLIPBOARD_GUI=qt4
     - python: 3.3
       env: NOSE_ARGS="not slow" FULL_DEPS=true CLIPBOARD=xsel
   exclude:
@@ -25,28 +25,25 @@ virtualenv:
   system_site_packages: true
 
 before_install:
-  - echo "Waldo1"
+  - echo "before_install"
   - echo $VIRTUAL_ENV
   - df -h
   - date
-  # - export PIP_ARGS=-q # comment this this to debug travis install issues
-  # - export APT_ARGS=-qq # comment this to debug travis install issues
-  # - set -x # enable this to see bash commands
-  - export ZIP_FLAGS=-q # comment this to debug travis install issues
   - ci/before_install.sh
   - python -V
+  # Xvfb stuff for clipboard functionality; see the travis-ci documentation
   - export DISPLAY=:99.0
   - sh -e /etc/init.d/xvfb start
 
 install:
-  - echo "Waldo2"
+  - echo "install"
   - ci/install.sh
 
 before_script:
   - mysql -e 'create database pandas_nosetest;'
 
 script:
-  - echo "Waldo3"
+  - echo "script"
   - ci/script.sh
 
 after_script:

ci/install.sh

@@ -13,20 +13,37 @@
 #    (no compiling needed), then directly goto script and collect 200$.
 #
 
-echo "inside $0"
+function edit_init()
+{
+    if [ -n "$LOCALE_OVERRIDE" ]; then
+        echo "Adding locale to the first line of pandas/__init__.py"
+        rm -f pandas/__init__.pyc
+        sedc="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LOCALE_OVERRIDE')\n"
+        sed -i "$sedc" pandas/__init__.py
+        echo "head -4 pandas/__init__.py"
+        head -4 pandas/__init__.py
+        echo
+    fi
+}
+
+edit_init
 
 # Install Dependencies
-# as of pip 1.4rc2, wheel files are still being broken regularly, this is a known good
-# commit. should revert to pypi when a final release is out
-pip install -I git+https://github.com/pypa/pip@42102e9deaea99db08b681d06906c2945f6f95e2#egg=pip
-pv="${TRAVIS_PYTHON_VERSION:0:1}"
-[ "$pv" == "2" ] && pv=""
+# as of pip 1.4rc2, wheel files are still being broken regularly, this is a
+# known good commit. should revert to pypi when a final release is out
+pip_commit=42102e9deaea99db08b681d06906c2945f6f95e2
+pip install -I git+https://github.com/pypa/pip@$pip_commit#egg=pip
+
+python_major_version="${TRAVIS_PYTHON_VERSION:0:1}"
+[ "$python_major_version" == "2" ] && python_major_version=""
 
 pip install -I -U setuptools
 pip install wheel
 
 # comment this line to disable the fetching of wheel files
-PIP_ARGS+=" -I --use-wheel --find-links=http://cache27diy-cpycloud.rhcloud.com/${TRAVIS_PYTHON_VERSION}${JOB_TAG}/"
+base_url=http://cache27diy-cpycloud.rhcloud.com
+wheel_box=${TRAVIS_PYTHON_VERSION}${JOB_TAG}
+PIP_ARGS+=" -I --use-wheel --find-links=$base_url/$wheel_box/"
 
 # Force virtualenv to accpet system_site_packages
 rm -f $VIRTUAL_ENV/lib/python$TRAVIS_PYTHON_VERSION/no-global-site-packages.txt
@@ -35,39 +52,53 @@ rm -f $VIRTUAL_ENV/lib/python$TRAVIS_PYTHON_VERSION/no-global-site-packages.txt
 if [ -n "$LOCALE_OVERRIDE" ]; then
     # make sure the locale is available
     # probably useless, since you would need to relogin
-    sudo locale-gen "$LOCALE_OVERRIDE"
+    time sudo locale-gen "$LOCALE_OVERRIDE"
 fi
 
-
 # show-skipped is working at this particular commit
-time pip install git+git://github.com/cpcloud/nose-show-skipped.git@fa4ff84e53c09247753a155b428c1bf2c69cb6c3
-time pip install $PIP_ARGS -r ci/requirements-${TRAVIS_PYTHON_VERSION}${JOB_TAG}.txt
-time sudo apt-get install libatlas-base-dev gfortran
+show_skipped_commit=fa4ff84e53c09247753a155b428c1bf2c69cb6c3
+time pip install git+git://github.com/cpcloud/nose-show-skipped.git@$show_skipped_commit
+time pip install $PIP_ARGS -r ci/requirements-${wheel_box}.txt
+
+# we need these for numpy
+time sudo apt-get $APT_ARGS install libatlas-base-dev gfortran
+
+
+# Need to enable for locale testing. The location of the locale file(s) is
+# distro specific. For example, on Arch Linux all of the locales are in a
+# commented file--/etc/locale.gen--that must be commented in to be used
+# whereas Ubuntu looks in /var/lib/locales/supported.d/* and generates locales
+# based on what's in the files in that folder
+time echo 'it_CH.UTF-8 UTF-8' | sudo tee -a /var/lib/locales/supported.d/it
+time sudo locale-gen
 
 
 # install gui for clipboard testing
-if [ -n "$GUI" ]; then
-    echo "Using GUI clipboard: $GUI"
-    [ -n "$pv" ] && py="py"
-    time sudo apt-get $APT_ARGS install python${pv}-${py}${GUI}
+if [ -n "$CLIPBOARD_GUI" ]; then
+    echo "Using CLIPBOARD_GUI: $CLIPBOARD_GUI"
+    [ -n "$python_major_version" ] && py="py"
+    python_cb_gui_pkg=python${python_major_version}-${py}${CLIPBOARD_GUI}
+    time sudo apt-get $APT_ARGS install $python_cb_gui_pkg
 fi
 
 
-# install a clipboard
+# install a clipboard if $CLIPBOARD is not empty
 if [ -n "$CLIPBOARD" ]; then
     echo "Using clipboard: $CLIPBOARD"
     time sudo apt-get $APT_ARGS install $CLIPBOARD
 fi
 
 
 # Optional Deps
-if [ x"$FULL_DEPS" == x"true" ]; then
+if [ -n "$FULL_DEPS" ]; then
     echo "Installing FULL_DEPS"
-    # for pytables gets the lib as well
+
+    # need libhdf5 for PyTables
     time sudo apt-get $APT_ARGS install libhdf5-serial-dev
 fi
 
-# build pandas
+
+# build and install pandas
 time python setup.py build_ext install
 
 true

ci/script.sh

@@ -5,8 +5,8 @@ echo "inside $0"
 if [ -n "$LOCALE_OVERRIDE" ]; then
     export LC_ALL="$LOCALE_OVERRIDE";
     echo "Setting LC_ALL to $LOCALE_OVERRIDE"
-    (cd /; python -c 'import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))')
-
+    pycmd='import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))'
+    python -c "$pycmd"
 fi
 
 echo nosetests --exe -w /tmp -A "$NOSE_ARGS" pandas --show-skipped

doc/source/release.rst

@@ -578,6 +578,9 @@ Bug Fixes
   - Fix a bug with ``NDFrame.replace()`` which made replacement appear as
     though it was (incorrectly) using regular expressions (:issue:`5143`).
   - Fix better error message for to_datetime (:issue:`4928`)
+  - Made sure different locales are tested on travis-ci (:issue:`4918`). Also
+    adds a couple of utilities for getting locales and setting locales with a
+    context manager.
 
 pandas 0.12.0
 -------------

pandas/io/tests/test_data.py

@@ -13,6 +13,7 @@
 from pandas.io.data import DataReader, SymbolWarning
 from pandas.util.testing import (assert_series_equal, assert_produces_warning,
                                  network, assert_frame_equal)
+import pandas.util.testing as tm
 from numpy.testing import assert_array_equal
 
 
@@ -35,6 +36,15 @@ def assert_n_failed_equals_n_null_columns(wngs, obj, cls=SymbolWarning):
 
 
 class TestGoogle(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.locales = tm.get_locales(prefix='en_US')
+        if not cls.locales:
+            raise nose.SkipTest("US English locale not available for testing")
+
+    @classmethod
+    def tearDownClass(cls):
+        del cls.locales
 
     @network
     def test_google(self):
@@ -44,9 +54,10 @@ def test_google(self):
         start = datetime(2010, 1, 1)
         end = datetime(2013, 1, 27)
 
-        self.assertEquals(
-            web.DataReader("F", 'google', start, end)['Close'][-1],
-            13.68)
+        for locale in self.locales:
+            with tm.set_locale(locale):
+                panel = web.DataReader("F", 'google', start, end)
+            self.assertEquals(panel.Close[-1], 13.68)
 
         self.assertRaises(Exception, web.DataReader, "NON EXISTENT TICKER",
                           'google', start, end)
@@ -58,38 +69,40 @@ def test_get_quote_fails(self):
 
     @network
     def test_get_goog_volume(self):
-        df = web.get_data_google('GOOG')
-        self.assertEqual(df.Volume.ix['OCT-08-2010'], 2863473)
+        for locale in self.locales:
+            with tm.set_locale(locale):
+                df = web.get_data_google('GOOG').sort_index()
+            self.assertEqual(df.Volume.ix['OCT-08-2010'], 2863473)
 
     @network
     def test_get_multi1(self):
-        sl = ['AAPL', 'AMZN', 'GOOG']
-        pan = web.get_data_google(sl, '2012')
-
-        def testit():
+        for locale in self.locales:
+            sl = ['AAPL', 'AMZN', 'GOOG']
+            with tm.set_locale(locale):
+                pan = web.get_data_google(sl, '2012')
             ts = pan.Close.GOOG.index[pan.Close.AAPL > pan.Close.GOOG]
-            self.assertEquals(ts[0].dayofyear, 96)
-
-        if (hasattr(pan, 'Close') and hasattr(pan.Close, 'GOOG') and
-            hasattr(pan.Close, 'AAPL')):
-            testit()
-        else:
-            self.assertRaises(AttributeError, testit)
+            if (hasattr(pan, 'Close') and hasattr(pan.Close, 'GOOG') and
+                hasattr(pan.Close, 'AAPL')):
+                self.assertEquals(ts[0].dayofyear, 96)
+            else:
+                self.assertRaises(AttributeError, lambda: pan.Close)
 
     @network
     def test_get_multi2(self):
         with warnings.catch_warnings(record=True) as w:
-            pan = web.get_data_google(['GE', 'MSFT', 'INTC'], 'JAN-01-12',
-                                      'JAN-31-12')
-            result = pan.Close.ix['01-18-12']
-            assert_n_failed_equals_n_null_columns(w, result)
-
-            # sanity checking
-
-            assert np.issubdtype(result.dtype, np.floating)
-            result = pan.Open.ix['Jan-15-12':'Jan-20-12']
-            self.assertEqual((4, 3), result.shape)
-            assert_n_failed_equals_n_null_columns(w, result)
+            for locale in self.locales:
+                with tm.set_locale(locale):
+                    pan = web.get_data_google(['GE', 'MSFT', 'INTC'],
+                                              'JAN-01-12', 'JAN-31-12')
+                result = pan.Close.ix['01-18-12']
+                assert_n_failed_equals_n_null_columns(w, result)
+
+                # sanity checking
+
+                assert np.issubdtype(result.dtype, np.floating)
+                result = pan.Open.ix['Jan-15-12':'Jan-20-12']
+                self.assertEqual((4, 3), result.shape)
+                assert_n_failed_equals_n_null_columns(w, result)
 
 
 class TestYahoo(unittest.TestCase):