Merge branch 'master' into feature/groupby-repr-ellipses-1135

Author: benjaminarjun

asv_bench/benchmarks/__init__.py

@@ -0,0 +1 @@
+"""Pandas benchmarks."""

asv_bench/benchmarks/io/hdf.py

@@ -1,7 +1,5 @@
-import warnings
-
 import numpy as np
-from pandas import DataFrame, Panel, date_range, HDFStore, read_hdf
+from pandas import DataFrame, date_range, HDFStore, read_hdf
 import pandas.util.testing as tm
 
 from ..pandas_vb_common import BaseIO
@@ -99,31 +97,6 @@ def time_store_info(self):
         self.store.info()
 
 
-class HDFStorePanel(BaseIO):
-
-    def setup(self):
-        self.fname = '__test__.h5'
-        with warnings.catch_warnings(record=True):
-            self.p = Panel(np.random.randn(20, 1000, 25),
-                           items=['Item%03d' % i for i in range(20)],
-                           major_axis=date_range('1/1/2000', periods=1000),
-                           minor_axis=['E%03d' % i for i in range(25)])
-            self.store = HDFStore(self.fname)
-            self.store.append('p1', self.p)
-
-    def teardown(self):
-        self.store.close()
-        self.remove(self.fname)
-
-    def time_read_store_table_panel(self):
-        with warnings.catch_warnings(record=True):
-            self.store.select('p1')
-
-    def time_write_store_table_panel(self):
-        with warnings.catch_warnings(record=True):
-            self.store.append('p2', self.p)
-
-
 class HDF(BaseIO):
 
     params = ['table', 'fixed']

asv_bench/benchmarks/series_methods.py

@@ -124,6 +124,25 @@ def time_dropna(self, dtype):
         self.s.dropna()
 
 
+class SearchSorted(object):
+
+    goal_time = 0.2
+    params = ['int8', 'int16', 'int32', 'int64',
+              'uint8', 'uint16', 'uint32', 'uint64',
+              'float16', 'float32', 'float64',
+              'str']
+    param_names = ['dtype']
+
+    def setup(self, dtype):
+        N = 10**5
+        data = np.array([1] * N + [2] * N + [3] * N).astype(dtype)
+        self.s = Series(data)
+
+    def time_searchsorted(self, dtype):
+        key = '2' if dtype == 'str' else 2
+        self.s.searchsorted(key)
+
+
 class Map(object):
 
     params = ['dict', 'Series']

ci/code_checks.sh

@@ -93,7 +93,7 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
     # this particular codebase (e.g. src/headers, src/klib, src/msgpack). However,
     # we can lint all header files since they aren't "generated" like C files are.
     MSG='Linting .c and .h' ; echo $MSG
-    cpplint --quiet --extensions=c,h --headers=h --recursive --filter=-readability/casting,-runtime/int,-build/include_subdir pandas/_libs/src/*.h pandas/_libs/src/parser pandas/_libs/ujson pandas/_libs/tslibs/src/datetime
+    cpplint --quiet --extensions=c,h --headers=h --recursive --filter=-readability/casting,-runtime/int,-build/include_subdir pandas/_libs/src/*.h pandas/_libs/src/parser pandas/_libs/ujson pandas/_libs/tslibs/src/datetime pandas/io/msgpack pandas/_libs/*.cpp pandas/util
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     echo "isort --version-number"
@@ -174,9 +174,10 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     MSG='Check that no file in the repo contains tailing whitespaces' ; echo $MSG
     set -o pipefail
     if [[ "$AZURE" == "true" ]]; then
-        ! grep -n --exclude="*.svg" -RI "\s$" * | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Tailing whitespaces found: " $3}'
+        # we exclude all c/cpp files as the c/cpp files of pandas code base are tested when Linting .c and .h files
+        ! grep -n '--exclude=*.'{svg,c,cpp,html} -RI "\s$" * | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Tailing whitespaces found: " $3}'
     else
-        ! grep -n --exclude="*.svg" -RI "\s$" * | awk -F ":" '{print $1 ":" $2 ":Tailing whitespaces found: " $3}'
+        ! grep -n '--exclude=*.'{svg,c,cpp,html}  -RI "\s$" * | awk -F ":" '{print $1 ":" $2 ":Tailing whitespaces found: " $3}'
     fi
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 fi
@@ -240,8 +241,8 @@ fi
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate docstrings (GL06, GL07, GL09, SS04, PR03, PR05, EX04, RT04, SS05)' ; echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=GL06,GL07,GL09,SS04,PR03,PR05,EX04,RT04,SS05
+    MSG='Validate docstrings (GL06, GL07, GL09, SS04, PR03, PR05, PR10, EX04, RT04, RT05, SS05, SA05)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=GL06,GL07,GL09,SS04,PR03,PR04,PR05,EX04,RT04,RT05,SS05,SA05
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/deps/azure-27-compat.yaml

@@ -20,6 +20,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - pytest-mock
   - pip:
     - html5lib==1.0b2
     - beautifulsoup4==4.2.1

ci/deps/azure-27-locale.yaml

@@ -22,6 +22,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - pytest-mock
   - hypothesis>=3.58.0
   - pip:
     - html5lib==1.0b2

ci/deps/azure-36-locale_slow.yaml

@@ -28,6 +28,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - pytest-mock
   - moto
   - pip:
     - hypothesis>=3.58.0

ci/deps/azure-37-locale.yaml

@@ -27,6 +27,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - pytest-mock
   - pip:
     - hypothesis>=3.58.0
     - moto  # latest moto in conda-forge fails with 3.7, move to conda dependencies when this is fixed

ci/deps/azure-37-numpydev.yaml

@@ -8,6 +8,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - pytest-mock
   - hypothesis>=3.58.0
   - pip:
     - "git+git://github.com/dateutil/dateutil.git"

ci/deps/azure-macos-35.yaml

@@ -24,6 +24,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - pytest-mock
   - pip:
     - python-dateutil==2.5.3
     - hypothesis>=3.58.0

ci/deps/azure-windows-27.yaml

@@ -27,5 +27,6 @@ dependencies:
   - cython>=0.28.2
   - pytest
   - pytest-xdist
+  - pytest-mock
   - moto
   - hypothesis>=3.58.0

ci/deps/azure-windows-36.yaml

@@ -25,4 +25,5 @@ dependencies:
   - cython>=0.28.2
   - pytest
   - pytest-xdist
+  - pytest-mock
   - hypothesis>=3.58.0

ci/deps/travis-27.yaml

@@ -41,6 +41,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - pytest-mock
   - moto==1.3.4
   - hypothesis>=3.58.0
   - pip:

ci/deps/travis-36-locale.yaml

@@ -30,6 +30,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - pytest-mock
   - moto
   - pip:
     - hypothesis>=3.58.0

ci/deps/travis-36-slow.yaml

@@ -27,5 +27,6 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - pytest-mock
   - moto
   - hypothesis>=3.58.0

ci/deps/travis-36.yaml

@@ -36,6 +36,7 @@ dependencies:
   - pytest
   - pytest-xdist
   - pytest-cov
+  - pytest-mock
   - hypothesis>=3.58.0
   - pip:
     - brotlipy

ci/deps/travis-37.yaml

@@ -14,6 +14,7 @@ dependencies:
   - pytz
   - pytest
   - pytest-xdist
+  - pytest-mock
   - hypothesis>=3.58.0
   - s3fs
   - pip:

doc/cheatsheet/Pandas_Cheat_Sheet.pdf

@@ -98,9 +98,9 @@
                 if (fname == 'index.rst'
                         and os.path.abspath(dirname) == source_path):
                     continue
-                elif pattern == '-api' and dirname == 'api':
+                elif pattern == '-api' and dirname == 'reference':
                     exclude_patterns.append(fname)
-                elif fname != pattern:
+                elif pattern != '-api' and fname != pattern:
                     exclude_patterns.append(fname)
 
 with open(os.path.join(source_path, 'index.rst.template')) as f:

doc/cheatsheet/Pandas_Cheat_Sheet.pptx

@@ -54,7 +54,7 @@ Bug reports must:
       ...
       ```
 
-#. Include the full version string of *pandas* and its dependencies. You can use the built in function::
+#. Include the full version string of *pandas* and its dependencies. You can use the built-in function::
 
       >>> import pandas as pd
       >>> pd.show_versions()
@@ -211,7 +211,7 @@ See the full conda docs `here <http://conda.pydata.org/docs>`__.
 Creating a Python Environment (pip)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-If you aren't using conda for you development environment, follow these instructions.
+If you aren't using conda for your development environment, follow these instructions.
 You'll need to have at least python3.5 installed on your system.
 
 .. code-block:: none
@@ -484,7 +484,7 @@ contributing them to the project::
 
    ./ci/code_checks.sh
 
-The script verify the linting of code files, it looks for common mistake patterns
+The script verifies the linting of code files, it looks for common mistake patterns
 (like missing spaces around sphinx directives that make the documentation not
 being rendered properly) and it also validates the doctests. It is possible to
 run the checks independently by using the parameters ``lint``, ``patterns`` and
@@ -675,7 +675,7 @@ Otherwise, you need to do it manually:
 
 You'll also need to
 
-1. write a new test that asserts a warning is issued when calling with the deprecated argument
+1. Write a new test that asserts a warning is issued when calling with the deprecated argument
 2. Update all of pandas existing tests and code to use the new argument
 
 See :ref:`contributing.warnings` for more.

doc/cheatsheet/Pandas_Cheat_Sheet_JA.pdf

@@ -33,8 +33,9 @@ decorate a class, providing the name of attribute to add. The class's
 
        @staticmethod
        def _validate(obj):
-           if 'lat' not in obj.columns or 'lon' not in obj.columns:
-               raise AttributeError("Must have 'lat' and 'lon'.")
+           # verify there is a column latitude and a column longitude
+           if 'latitude' not in obj.columns or 'longitude' not in obj.columns:
+               raise AttributeError("Must have 'latitude' and 'longitude'.")
 
        @property
        def center(self):

doc/cheatsheet/Pandas_Cheat_Sheet_JA.pptx

@@ -505,7 +505,7 @@ So, for instance, to reproduce :meth:`~DataFrame.combine_first` as above:
 .. ipython:: python
 
    def combiner(x, y):
-       np.where(pd.isna(x), y, x)
+       return np.where(pd.isna(x), y, x)
    df1.combine(df2, combiner)
 
 .. _basics.stats:

doc/source/conf.py

@@ -120,6 +120,7 @@ Methods
    Timestamp.timetuple
    Timestamp.timetz
    Timestamp.to_datetime64
+   Timestamp.to_numpy
    Timestamp.to_julian_date
    Timestamp.to_period
    Timestamp.to_pydatetime
@@ -191,6 +192,7 @@ Methods
    Timedelta.round
    Timedelta.to_pytimedelta
    Timedelta.to_timedelta64
+   Timedelta.to_numpy
    Timedelta.total_seconds
 
 A collection of timedeltas may be stored in a :class:`TimedeltaArray`.

doc/source/development/contributing.rst

@@ -1317,7 +1317,7 @@ arbitrary function, for example:
    df.groupby(['Store', 'Product']).pipe(mean)
 
 where ``mean`` takes a GroupBy object and finds the mean of the Revenue and Quantity
-columns repectively for each Store-Product combination. The ``mean`` function can
+columns respectively for each Store-Product combination. The ``mean`` function can
 be any function that takes in a GroupBy object; the ``.pipe`` will pass the GroupBy
 object as a parameter into the function you specify.
 

doc/source/development/extending.rst

@@ -335,7 +335,7 @@ examined :ref:`in the API <api.dataframe.missing>`.
 Interpolation
 ~~~~~~~~~~~~~
 
-.. versionadded:: 0.21.0
+.. versionadded:: 0.23.0
 
   The ``limit_area`` keyword argument was added.
 

doc/source/getting_started/basics.rst

@@ -321,6 +321,15 @@ which can be specified. These are computed from the starting point specified by
    pd.to_datetime([1349720105100, 1349720105200, 1349720105300,
                    1349720105400, 1349720105500], unit='ms')
 
+Constructing a :class:`Timestamp` or :class:`DatetimeIndex` with an epoch timestamp
+with the ``tz`` argument specified will localize the epoch timestamps to UTC
+first then convert the result to the specified time zone.
+
+.. ipython:: python
+
+   pd.Timestamp(1262347200000000000, tz='US/Pacific')
+   pd.DatetimeIndex([1262347200000000000], tz='US/Pacific')
+
 .. note::
 
    Epoch times will be rounded to the nearest nanosecond.
@@ -624,6 +633,16 @@ We are stopping on the included end-point as it is part of the index:
    dft2 = dft2.swaplevel(0, 1).sort_index()
    dft2.loc[idx[:, '2013-01-05'], :]
 
+.. versionadded:: 0.25.0
+
+Slicing with string indexing also honors UTC offset.
+
+.. ipython:: python
+
+    df = pd.DataFrame([0], index=pd.DatetimeIndex(['2019-01-01'], tz='US/Pacific'))
+    df
+    df['2019-01-01 12:00:00+04:00':'2019-01-01 13:00:00+04:00']
+
 .. _timeseries.slice_vs_exact_match:
 
 Slice vs. Exact Match
@@ -2205,6 +2224,21 @@ you can use the ``tz_convert`` method.
 
    rng_pytz.tz_convert('US/Eastern')
 
+.. note::
+
+    When using ``pytz`` time zones, :class:`DatetimeIndex` will construct a different
+    time zone object than a :class:`Timestamp` for the same time zone input. A :class:`DatetimeIndex`
+    can hold a collection of :class:`Timestamp` objects that may have different UTC offsets and cannot be
+    succinctly represented by one ``pytz`` time zone instance while one :class:`Timestamp`
+    represents one point in time with a specific UTC offset.
+
+    .. ipython:: python
+
+       dti = pd.date_range('2019-01-01', periods=3, freq='D', tz='US/Pacific')
+       dti.tz
+       ts = pd.Timestamp('2019-01-01', tz='US/Pacific')
+       ts.tz
+
 .. warning::
 
 	Be wary of conversions between libraries. For some time zones, ``pytz`` and ``dateutil`` have different

doc/source/reference/arrays.rst

@@ -370,7 +370,7 @@ Updated PyTables Support
       df1.get_dtype_counts()
 
 - performance improvements on table writing
-- support for arbitrarly indexed dimensions
+- support for arbitrarily indexed dimensions
 - ``SparseSeries`` now has a ``density`` property (:issue:`2384`)
 - enable ``Series.str.strip/lstrip/rstrip`` methods to take an input argument
   to strip arbitrary characters (:issue:`2411`)

doc/source/styled.xlsx

@@ -136,7 +136,7 @@ groupby operations on the index will preserve the index nature as well
 reindexing operations, will return a resulting index based on the type of the passed
 indexer, meaning that passing a list will return a plain-old-``Index``; indexing with
 a ``Categorical`` will return a ``CategoricalIndex``, indexed according to the categories
-of the PASSED ``Categorical`` dtype. This allows one to arbitrarly index these even with
+of the PASSED ``Categorical`` dtype. This allows one to arbitrarily index these even with
 values NOT in the categories, similarly to how you can reindex ANY pandas index.
 
 .. code-block:: ipython