Add test for timedeltas data

Author: LucasG0

asv_bench/benchmarks/arithmetic.py

@@ -466,7 +466,7 @@ def setup(self, offset):
         self.rng = rng
 
     def time_apply_index(self, offset):
-        offset.apply_index(self.rng)
+        self.rng + offset
 
 
 class BinaryOpsMultiIndex:

asv_bench/benchmarks/io/json.py

@@ -53,12 +53,18 @@ def time_read_json_lines(self, index):
     def time_read_json_lines_concat(self, index):
         concat(read_json(self.fname, orient="records", lines=True, chunksize=25000))
 
+    def time_read_json_lines_nrows(self, index):
+        read_json(self.fname, orient="records", lines=True, nrows=25000)
+
     def peakmem_read_json_lines(self, index):
         read_json(self.fname, orient="records", lines=True)
 
     def peakmem_read_json_lines_concat(self, index):
         concat(read_json(self.fname, orient="records", lines=True, chunksize=25000))
 
+    def peakmem_read_json_lines_nrows(self, index):
+        read_json(self.fname, orient="records", lines=True, nrows=15000)
+
 
 class ToJSON(BaseIO):
 

ci/deps/travis-36-locale.yaml

@@ -27,7 +27,7 @@ dependencies:
   - numexpr
   - numpy
   - openpyxl
-  - pandas-gbq=0.8.0
+  - pandas-gbq=0.12.0
   - psycopg2=2.6.2
   - pymysql=0.7.11
   - pytables

doc/source/development/contributing.rst

@@ -270,7 +270,7 @@ Creating a Python environment (pip)
 If you aren't using conda for your development environment, follow these instructions.
 You'll need to have at least Python 3.6.1 installed on your system.
 
-**Unix**/**Mac OS**
+**Unix**/**Mac OS with virtualenv**
 
 .. code-block:: bash
 
@@ -286,7 +286,31 @@ You'll need to have at least Python 3.6.1 installed on your system.
    python -m pip install -r requirements-dev.txt
 
    # Build and install pandas
-   python setup.py build_ext --inplace -j 0
+   python setup.py build_ext --inplace -j 4
+   python -m pip install -e . --no-build-isolation --no-use-pep517
+
+**Unix**/**Mac OS with pyenv**
+
+Consult the docs for setting up pyenv `here <https://github.com/pyenv/pyenv>`__.
+
+.. code-block:: bash
+
+   # Create a virtual environment
+   # Use an ENV_DIR of your choice. We'll use ~/Users/<yourname>/.pyenv/versions/pandas-dev
+
+   pyenv virtualenv <version> <name-to-give-it>
+
+   # For instance:
+   pyenv virtualenv 3.7.6 pandas-dev
+
+   # Activate the virtualenv
+   pyenv activate pandas-dev
+
+   # Now install the build dependencies in the cloned pandas repo
+   python -m pip install -r requirements-dev.txt
+
+   # Build and install pandas
+   python setup.py build_ext --inplace -j 4
    python -m pip install -e . --no-build-isolation --no-use-pep517
 
 **Windows**
@@ -312,7 +336,7 @@ should already exist.
    python -m pip install -r requirements-dev.txt
 
    # Build and install pandas
-   python setup.py build_ext --inplace -j 0
+   python setup.py build_ext --inplace -j 4
    python -m pip install -e . --no-build-isolation --no-use-pep517
 
 Creating a branch

doc/source/getting_started/comparison/comparison_with_sas.rst

@@ -115,7 +115,7 @@ Reading external data
 
 Like SAS, pandas provides utilities for reading in data from
 many formats.  The ``tips`` dataset, found within the pandas
-tests (`csv <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/data/tips.csv>`_)
+tests (`csv <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/io/data/csv/tips.csv>`_)
 will be used in many of the following examples.
 
 SAS provides ``PROC IMPORT`` to read csv data into a data set.
@@ -131,7 +131,7 @@ The pandas method is :func:`read_csv`, which works similarly.
 .. ipython:: python
 
    url = ('https://raw.github.com/pandas-dev/'
-          'pandas/master/pandas/tests/data/tips.csv')
+          'pandas/master/pandas/tests/io/data/csv/tips.csv')
    tips = pd.read_csv(url)
    tips.head()
 

doc/source/getting_started/comparison/comparison_with_sql.rst

@@ -25,7 +25,7 @@ structure.
 .. ipython:: python
 
     url = ('https://raw.github.com/pandas-dev'
-           '/pandas/master/pandas/tests/data/tips.csv')
+           '/pandas/master/pandas/tests/io/data/csv/tips.csv')
     tips = pd.read_csv(url)
     tips.head()
 

doc/source/getting_started/comparison/comparison_with_stata.rst

@@ -112,7 +112,7 @@ Reading external data
 
 Like Stata, pandas provides utilities for reading in data from
 many formats.  The ``tips`` data set, found within the pandas
-tests (`csv <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/data/tips.csv>`_)
+tests (`csv <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/io/data/csv/tips.csv>`_)
 will be used in many of the following examples.
 
 Stata provides ``import delimited`` to read csv data into a data set in memory.
@@ -128,7 +128,7 @@ the data set if presented with a url.
 .. ipython:: python
 
    url = ('https://raw.github.com/pandas-dev'
-          '/pandas/master/pandas/tests/data/tips.csv')
+          '/pandas/master/pandas/tests/io/data/csv/tips.csv')
    tips = pd.read_csv(url)
    tips.head()
 

doc/source/getting_started/install.rst

@@ -274,7 +274,7 @@ lxml                      3.8.0              HTML parser for read_html (see :ref
 matplotlib                2.2.2              Visualization
 numba                     0.46.0             Alternative execution engine for rolling operations
 openpyxl                  2.5.7              Reading / writing for xlsx files
-pandas-gbq                0.8.0              Google Big Query access
+pandas-gbq                0.12.0             Google Big Query access
 psycopg2                                     PostgreSQL engine for sqlalchemy
 pyarrow                   0.12.0             Parquet, ORC (requires 0.13.0), and feather reading / writing
 pymysql                   0.7.11             MySQL engine for sqlalchemy

doc/source/reference/frame.rst

@@ -47,8 +47,6 @@ Conversion
    DataFrame.convert_dtypes
    DataFrame.infer_objects
    DataFrame.copy
-   DataFrame.isna
-   DataFrame.notna
    DataFrame.bool
 
 Indexing, iteration
@@ -211,10 +209,18 @@ Missing data handling
 .. autosummary::
    :toctree: api/
 
+   DataFrame.backfill
+   DataFrame.bfill
    DataFrame.dropna
+   DataFrame.ffill
    DataFrame.fillna
-   DataFrame.replace
    DataFrame.interpolate
+   DataFrame.isna
+   DataFrame.isnull
+   DataFrame.notna
+   DataFrame.notnull
+   DataFrame.pad
+   DataFrame.replace
 
 Reshaping, sorting, transposing
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

doc/source/reference/groupby.rst

@@ -50,6 +50,7 @@ Computations / descriptive stats
    GroupBy.all
    GroupBy.any
    GroupBy.bfill
+   GroupBy.backfill
    GroupBy.count
    GroupBy.cumcount
    GroupBy.cummax
@@ -67,6 +68,7 @@ Computations / descriptive stats
    GroupBy.ngroup
    GroupBy.nth
    GroupBy.ohlc
+   GroupBy.pad
    GroupBy.prod
    GroupBy.rank
    GroupBy.pct_change
@@ -88,10 +90,12 @@ application to columns of a specific data type.
 
    DataFrameGroupBy.all
    DataFrameGroupBy.any
+   DataFrameGroupBy.backfill
    DataFrameGroupBy.bfill
    DataFrameGroupBy.corr
    DataFrameGroupBy.count
    DataFrameGroupBy.cov
+   DataFrameGroupBy.cumcount
    DataFrameGroupBy.cummax
    DataFrameGroupBy.cummin
    DataFrameGroupBy.cumprod
@@ -106,6 +110,7 @@ application to columns of a specific data type.
    DataFrameGroupBy.idxmin
    DataFrameGroupBy.mad
    DataFrameGroupBy.nunique
+   DataFrameGroupBy.pad
    DataFrameGroupBy.pct_change
    DataFrameGroupBy.plot
    DataFrameGroupBy.quantile

doc/source/reference/series.rst

@@ -214,11 +214,18 @@ Missing data handling
 .. autosummary::
    :toctree: api/
 
-   Series.isna
-   Series.notna
+   Series.backfill
+   Series.bfill
    Series.dropna
+   Series.ffill
    Series.fillna
    Series.interpolate
+   Series.isna
+   Series.isnull
+   Series.notna
+   Series.notnull
+   Series.pad
+   Series.replace
 
 Reshaping, sorting
 ------------------

doc/source/user_guide/merging.rst

@@ -1273,7 +1273,7 @@ columns:
 
 .. ipython:: python
 
-   result = pd.merge(left, right, on='k', suffixes=['_l', '_r'])
+   result = pd.merge(left, right, on='k', suffixes=('_l', '_r'))
 
 .. ipython:: python
    :suppress:

doc/source/user_guide/visualization.rst

@@ -865,7 +865,7 @@ for more information. By coloring these curves differently for each class
 it is possible to visualize data clustering. Curves belonging to samples
 of the same class will usually be closer together and form larger structures.
 
-**Note**: The "Iris" dataset is available `here <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/data/iris.csv>`__.
+**Note**: The "Iris" dataset is available `here <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/io/data/csv/iris.csv>`__.
 
 .. ipython:: python
 
@@ -1025,7 +1025,7 @@ be colored differently.
 See the R package `Radviz <https://cran.r-project.org/package=Radviz/>`__
 for more information.
 
-**Note**: The "Iris" dataset is available `here <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/data/iris.csv>`__.
+**Note**: The "Iris" dataset is available `here <https://raw.github.com/pandas-dev/pandas/master/pandas/tests/io/data/csv/iris.csv>`__.
 
 .. ipython:: python
 
@@ -1332,7 +1332,7 @@ otherwise you will see a warning.
 
 .. ipython:: python
 
-   fig, axes = plt.subplots(4, 4, figsize=(6, 6))
+   fig, axes = plt.subplots(4, 4, figsize=(9, 9))
    plt.subplots_adjust(wspace=0.5, hspace=0.5)
    target1 = [axes[0][0], axes[1][1], axes[2][2], axes[3][3]]
    target2 = [axes[3][0], axes[2][1], axes[1][2], axes[0][3]]
@@ -1369,6 +1369,7 @@ Another option is passing an ``ax`` argument to :meth:`Series.plot` to plot on a
 .. ipython:: python
 
    fig, axes = plt.subplots(nrows=2, ncols=2)
+   plt.subplots_adjust(wspace=0.2, hspace=0.5)
    df['A'].plot(ax=axes[0, 0]);
    axes[0, 0].set_title('A');
    df['B'].plot(ax=axes[0, 1]);

doc/source/whatsnew/v1.1.0.rst

@@ -288,6 +288,9 @@ Other enhancements
 - :meth:`HDFStore.put` now accepts `track_times` parameter. Parameter is passed to ``create_table`` method of ``PyTables`` (:issue:`32682`).
 - Make :class:`pandas.core.window.Rolling` and :class:`pandas.core.window.Expanding` iterable（:issue:`11704`)
 - Make ``option_context`` a :class:`contextlib.ContextDecorator`, which allows it to be used as a decorator over an entire function (:issue:`34253`).
+- :meth:`groupby.transform` now allows ``func`` to be ``pad``, ``backfill`` and ``cumcount`` (:issue:`31269`).
+- :meth:`~pandas.io.json.read_json` now accepts `nrows` parameter. (:issue:`33916`).
+- :meth `~pandas.io.gbq.read_gbq` now allows to disable progress bar (:issue:`33360`).
 
 .. ---------------------------------------------------------------------------
 
@@ -354,6 +357,8 @@ Optional libraries below the lowest tested version may still work, but are not c
 +-----------------+-----------------+---------+
 | xlwt            | 1.2.0           |         |
 +-----------------+-----------------+---------+
+| pandas-gbq      | 1.2.0           |    X    |
++-----------------+-----------------+---------+
 
 See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more.
 
@@ -396,6 +401,7 @@ Backwards incompatible API changes
 - :func: `pandas.api.dtypes.is_string_dtype` no longer incorrectly identifies categorical series as string.
 - :func:`read_excel` no longer takes ``**kwds`` arguments. This means that passing in keyword ``chunksize`` now raises a ``TypeError``
   (previously raised a ``NotImplementedError``), while passing in keyword ``encoding`` now raises a ``TypeError`` (:issue:`34464`)
+- :func: `merge` now checks ``suffixes`` parameter type to be ``tuple`` and raises ``TypeError``, whereas before a ``list`` or ``set`` were accepted and that the ``set`` could produce unexpected results (:issue:`33740`)
 
 ``MultiIndex.get_indexer`` interprets `method` argument differently
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -821,6 +827,7 @@ Timedelta
 - Bug in :func:`timedelta_range` that produced an extra point on a edge case (:issue:`30353`, :issue:`33498`)
 - Bug in :meth:`DataFrame.resample` that produced an extra point on a edge case (:issue:`30353`, :issue:`13022`, :issue:`33498`)
 - Bug in :meth:`DataFrame.resample` that ignored the ``loffset`` argument when dealing with timedelta (:issue:`7687`, :issue:`33498`)
+- Bug in :class:`Timedelta` and `pandas.to_timedelta` that ignored `unit`-argument for string input (:issue:`12136`)
 
 Timezones
 ^^^^^^^^^
@@ -887,6 +894,7 @@ Indexing
 - Bug in :meth:`DataFrame.truncate` and :meth:`Series.truncate` where index was assumed to be monotone increasing (:issue:`33756`)
 - Indexing with a list of strings representing datetimes failed on :class:`DatetimeIndex` or :class:`PeriodIndex`(:issue:`11278`)
 - Bug in :meth:`Series.at` when used with a :class:`MultiIndex` would raise an exception on valid inputs (:issue:`26989`)
+- Bug in :meth:`Series.loc` when used with a :class:`MultiIndex` would raise an IndexingError when accessing a None value (:issue:`34318`)
 
 Missing
 ^^^^^^^
@@ -977,6 +985,7 @@ Groupby/resample/rolling
   to the input DataFrame is inconsistent. An internal heuristic to detect index mutation would behave differently for equal but not identical
   indices. In particular, the result index shape might change if a copy of the input would be returned.
   The behaviour now is consistent, independent of internal heuristics. (:issue:`31612`, :issue:`14927`, :issue:`13056`)
+- Bug in :meth:`SeriesGroupBy.agg` where any column name was accepted in the named aggregation of ``SeriesGroupBy`` previously. The behaviour now allows only ``str`` and callables else would raise ``TypeError``. (:issue:`34422`)
 
 Reshaping
 ^^^^^^^^^
@@ -1005,7 +1014,9 @@ Reshaping
 - Bug in :func:`concat` was not allowing for concatenation of ``DataFrame`` and ``Series`` with duplicate keys (:issue:`33654`)
 - Bug in :func:`cut` raised an error when non-unique labels (:issue:`33141`)
 - Ensure only named functions can be used in :func:`eval()` (:issue:`32460`)
+- Bug in :func:`Dataframe.aggregate` and :func:`Series.aggregate` was causing recursive loop in some cases (:issue:`34224`)
 - Fixed bug in :func:`melt` where melting MultiIndex columns with ``col_level`` > 0 would raise a ``KeyError`` on ``id_vars`` (:issue:`34129`)
+- Bug in :meth:`Series.where` with an empty Series and empty ``cond`` having non-bool dtype (:issue:`34592`)
 
 Sparse
 ^^^^^^

pandas/_libs/index.pyx

@@ -22,9 +22,9 @@ cnp.import_array()
 from pandas._libs cimport util
 
 from pandas._libs.tslibs.nattype cimport c_NaT as NaT
-from pandas._libs.tslibs.base cimport ABCTimedelta
 from pandas._libs.tslibs.period cimport is_period_object
 from pandas._libs.tslibs.timestamps cimport _Timestamp
+from pandas._libs.tslibs.timedeltas cimport _Timedelta
 
 from pandas._libs.hashtable cimport HashTable
 
@@ -471,7 +471,7 @@ cdef class TimedeltaEngine(DatetimeEngine):
         return 'm8[ns]'
 
     cdef int64_t _unbox_scalar(self, scalar) except? -1:
-        if not (isinstance(scalar, ABCTimedelta) or scalar is NaT):
+        if not (isinstance(scalar, _Timedelta) or scalar is NaT):
             raise TypeError(scalar)
         return scalar.value
 

pandas/_libs/interval.pyx

@@ -42,9 +42,9 @@ from pandas._libs.tslibs.util cimport (
     is_timedelta64_object,
 )
 
-from pandas._libs.tslibs.base cimport ABCTimedelta
 from pandas._libs.tslibs.timezones cimport tz_compare
 from pandas._libs.tslibs.timestamps cimport _Timestamp
+from pandas._libs.tslibs.timedeltas cimport _Timedelta
 
 _VALID_CLOSED = frozenset(['left', 'right', 'both', 'neither'])
 
@@ -340,7 +340,7 @@ cdef class Interval(IntervalMixin):
     def _validate_endpoint(self, endpoint):
         # GH 23013
         if not (is_integer_object(endpoint) or is_float_object(endpoint) or
-                isinstance(endpoint, (_Timestamp, ABCTimedelta))):
+                isinstance(endpoint, (_Timestamp, _Timedelta))):
             raise ValueError("Only numeric, Timestamp and Timedelta endpoints "
                              "are allowed when constructing an Interval.")
 

pandas/_libs/tslibs/__init__.py

@@ -1,4 +1,5 @@
 __all__ = [
+    "dtypes",
     "localize_pydatetime",
     "NaT",
     "NaTType",
@@ -14,12 +15,14 @@
     "ints_to_pytimedelta",
     "Timestamp",
     "tz_convert_single",
+    "to_offset",
 ]
 
-
+from . import dtypes  # type: ignore
 from .conversion import localize_pydatetime
 from .nattype import NaT, NaTType, iNaT, is_null_datetimelike, nat_strings
 from .np_datetime import OutOfBoundsDatetime
+from .offsets import to_offset
 from .period import IncompatibleFrequency, Period
 from .resolution import Resolution
 from .timedeltas import Timedelta, delta_to_nanoseconds, ints_to_pytimedelta