Merge branch 'master' into enh-loc-datetime-list

Author: proost

asv_bench/benchmarks/frame_methods.py

@@ -564,7 +564,7 @@ def setup(self):
 
     def time_frame_get_dtype_counts(self):
         with warnings.catch_warnings(record=True):
-            self.df._data.get_dtype_counts()
+            self.df.dtypes.value_counts()
 
     def time_info(self):
         self.df.info()

asv_bench/benchmarks/groupby.py

@@ -626,4 +626,38 @@ def time_first(self):
         self.df_nans.groupby("key").transform("first")
 
 
+class TransformEngine:
+    def setup(self):
+        N = 10 ** 3
+        data = DataFrame(
+            {0: [str(i) for i in range(100)] * N, 1: list(range(100)) * N},
+            columns=[0, 1],
+        )
+        self.grouper = data.groupby(0)
+
+    def time_series_numba(self):
+        def function(values, index):
+            return values * 5
+
+        self.grouper[1].transform(function, engine="numba")
+
+    def time_series_cython(self):
+        def function(values):
+            return values * 5
+
+        self.grouper[1].transform(function, engine="cython")
+
+    def time_dataframe_numba(self):
+        def function(values, index):
+            return values * 5
+
+        self.grouper.transform(function, engine="numba")
+
+    def time_dataframe_cython(self):
+        def function(values):
+            return values * 5
+
+        self.grouper.transform(function, engine="cython")
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

ci/deps/azure-36-minimum_versions.yaml

@@ -21,7 +21,7 @@ dependencies:
   - numexpr=2.6.2
   - numpy=1.13.3
   - openpyxl=2.5.7
-  - pytables=3.4.2
+  - pytables=3.4.3
   - python-dateutil=2.7.3
   - pytz=2017.2
   - scipy=0.19.0

ci/deps/azure-37-numpydev.yaml

@@ -14,7 +14,8 @@ dependencies:
   - pytz
   - pip
   - pip:
-    - cython>=0.29.16
+    - cython==0.29.16
+    # GH#33507 cython 3.0a1 is causing TypeErrors 2020-04-13
     - "git+git://github.com/dateutil/dateutil.git"
     - "-f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"
     - "--pre"

doc/source/getting_started/index.rst

@@ -398,7 +398,7 @@ data set, a sliding window of the data or grouped by categories. The latter is a
                 <div class="card-body">
 
 Change the structure of your data table in multiple ways. You can :func:`~pandas.melt` your data table from wide to long/tidy form or :func:`~pandas.pivot`
-from long to wide format. With aggregations built-in, a pivot table is created with a sinlge command.
+from long to wide format. With aggregations built-in, a pivot table is created with a single command.
 
 .. image:: ../_static/schemas/07_melt.svg
    :align: center

doc/source/getting_started/install.rst

@@ -262,7 +262,7 @@ BeautifulSoup4            4.6.0              HTML parser for read_html (see :ref
 Jinja2                                       Conditional formatting with DataFrame.style
 PyQt4                                        Clipboard I/O
 PyQt5                                        Clipboard I/O
-PyTables                  3.4.2              HDF5-based reading / writing
+PyTables                  3.4.3              HDF5-based reading / writing
 SQLAlchemy                1.1.4              SQL support for databases other than sqlite
 SciPy                     0.19.0             Miscellaneous statistical functions
 XLsxWriter                0.9.8              Excel writing
@@ -279,7 +279,7 @@ psycopg2                                     PostgreSQL engine for sqlalchemy
 pyarrow                   0.12.0             Parquet, ORC (requires 0.13.0), and feather reading / writing
 pymysql                   0.7.11             MySQL engine for sqlalchemy
 pyreadstat                                   SPSS files (.sav) reading
-pytables                  3.4.2              HDF5 reading / writing
+pytables                  3.4.3              HDF5 reading / writing
 pyxlsb                    1.0.6              Reading for xlsb files
 qtpy                                         Clipboard I/O
 s3fs                      0.3.0              Amazon S3 access

doc/source/getting_started/intro_tutorials/03_subset_data.rst

@@ -23,7 +23,7 @@
                     <div class="card-body">
                         <p class="card-text">
 
-This tutorial uses the titanic data set, stored as CSV. The data
+This tutorial uses the Titanic data set, stored as CSV. The data
 consists of the following data columns:
 
 -  PassengerId: Id of every passenger.
@@ -72,7 +72,7 @@ How do I select specific columns from a ``DataFrame``?
     <ul class="task-bullet">
         <li>
 
-I’m interested in the age of the titanic passengers.
+I’m interested in the age of the Titanic passengers.
 
 .. ipython:: python
 
@@ -111,7 +111,7 @@ the number of rows is returned.
     <ul class="task-bullet">
         <li>
 
-I’m interested in the age and sex of the titanic passengers.
+I’m interested in the age and sex of the Titanic passengers.
 
 .. ipython:: python
 
@@ -198,7 +198,7 @@ can be used to filter the ``DataFrame`` by putting it in between the
 selection brackets ``[]``. Only rows for which the value is ``True``
 will be selected.
 
-We now from before that the original titanic ``DataFrame`` consists of
+We know from before that the original Titanic ``DataFrame`` consists of
 891 rows. Let’s have a look at the amount of rows which satisfy the
 condition by checking the ``shape`` attribute of the resulting
 ``DataFrame`` ``above_35``:
@@ -212,7 +212,7 @@ condition by checking the ``shape`` attribute of the resulting
     <ul class="task-bullet">
         <li>
 
-I’m interested in the titanic passengers from cabin class 2 and 3.
+I’m interested in the Titanic passengers from cabin class 2 and 3.
 
 .. ipython:: python
 

doc/source/getting_started/intro_tutorials/10_text_data.rst

@@ -199,7 +199,7 @@ names in the ``Name`` column. By using pandas string methods, the
 
 Next, we need to get the corresponding location, preferably the index
 label, in the table for which the name length is the largest. The
-:meth:`~Series.idxmax`` method does exactly that. It is not a string method and is
+:meth:`~Series.idxmax` method does exactly that. It is not a string method and is
 applied to integers, so no ``str`` is used.
 
 .. ipython:: python

doc/source/user_guide/computation.rst

@@ -312,15 +312,35 @@ We provide a number of common statistical functions:
     :meth:`~Rolling.median`, Arithmetic median of values
     :meth:`~Rolling.min`, Minimum
     :meth:`~Rolling.max`, Maximum
-    :meth:`~Rolling.std`, Bessel-corrected sample standard deviation
-    :meth:`~Rolling.var`, Unbiased variance
+    :meth:`~Rolling.std`, Sample standard deviation
+    :meth:`~Rolling.var`, Sample variance
     :meth:`~Rolling.skew`, Sample skewness (3rd moment)
     :meth:`~Rolling.kurt`, Sample kurtosis (4th moment)
     :meth:`~Rolling.quantile`, Sample quantile (value at %)
     :meth:`~Rolling.apply`, Generic apply
     :meth:`~Rolling.cov`, Unbiased covariance (binary)
     :meth:`~Rolling.corr`, Correlation (binary)
 
+.. _computation.window_variance.caveats:
+
+.. note::
+
+   Please note that :meth:`~Rolling.std` and :meth:`~Rolling.var` use the sample
+   variance formula by default, i.e. the sum of squared differences is divided by
+   ``window_size - 1`` and not by ``window_size`` during averaging. In statistics,
+   we use sample when the dataset is drawn from a larger population that we
+   don't have access to. Using it implies that the data in our window is a
+   random sample from the population, and we are interested not in the variance
+   inside the specific window but in the variance of some general window that
+   our windows represent. In this situation, using the sample variance formula
+   results in an unbiased estimator and so is preferred.
+
+   Usually, we are instead interested in the variance of each window as we slide
+   it over the data, and in this case we should specify ``ddof=0`` when calling
+   these methods to use population variance instead of sample variance. Using
+   sample variance under the circumstances would result in a biased estimator
+   of the variable we are trying to determine.
+
 .. _stats.rolling_apply:
 
 Rolling apply
@@ -848,15 +868,22 @@ Method summary
     :meth:`~Expanding.median`, Arithmetic median of values
     :meth:`~Expanding.min`, Minimum
     :meth:`~Expanding.max`, Maximum
-    :meth:`~Expanding.std`, Unbiased standard deviation
-    :meth:`~Expanding.var`, Unbiased variance
+    :meth:`~Expanding.std`, Sample standard deviation
+    :meth:`~Expanding.var`, Sample variance
     :meth:`~Expanding.skew`, Unbiased skewness (3rd moment)
     :meth:`~Expanding.kurt`, Unbiased kurtosis (4th moment)
     :meth:`~Expanding.quantile`, Sample quantile (value at %)
     :meth:`~Expanding.apply`, Generic apply
     :meth:`~Expanding.cov`, Unbiased covariance (binary)
     :meth:`~Expanding.corr`, Correlation (binary)
 
+.. note::
+
+   Using sample variance formulas for :meth:`~Expanding.std` and
+   :meth:`~Expanding.var` comes with the same caveats as using them with rolling
+   windows. See :ref:`this section <computation.window_variance.caveats>` for more
+   information.
+
 .. currentmodule:: pandas
 
 Aside from not having a ``window`` parameter, these functions have the same

doc/source/user_guide/timeseries.rst

@@ -786,6 +786,15 @@ Furthermore, if you have a ``Series`` with datetimelike values, then you can
 access these properties via the ``.dt`` accessor, as detailed in the section
 on :ref:`.dt accessors<basics.dt_accessors>`.
 
+.. versionadded:: 1.1.0
+
+You may obtain the year, week and day components of the ISO year from the ISO 8601 standard:
+
+.. ipython:: python
+
+   idx = pd.date_range(start='2019-12-29', freq='D', periods=4)
+   idx.to_series().dt.isocalendar()
+
 .. _timeseries.offsets:
 
 DateOffset objects

doc/source/whatsnew/v0.14.0.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0140:
 
-v0.14.0 (May 31 , 2014)
------------------------
+Version 0.14.0 (May 31 , 2014)
+------------------------------
 
 {{ header }}
 
@@ -321,7 +321,7 @@ Text parsing API changes
 
 .. _whatsnew_0140.groupby:
 
-Groupby API changes
+GroupBy API changes
 ~~~~~~~~~~~~~~~~~~~
 
 More consistent behavior for some groupby methods:
@@ -473,8 +473,8 @@ Some other enhancements to the sql functions include:
 
 .. _whatsnew_0140.slicers:
 
-Multiindexing using slicers
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Multi-indexing using slicers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 In 0.14.0 we added a new way to slice MultiIndexed objects.
 You can slice a MultiIndex by providing multiple indexers.

doc/source/whatsnew/v0.14.1.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0141:
 
-v0.14.1 (July 11, 2014)
------------------------
+Version 0.14.1 (July 11, 2014)
+------------------------------
 
 {{ header }}
 

doc/source/whatsnew/v0.15.0.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0150:
 
-v0.15.0 (October 18, 2014)
---------------------------
+Version 0.15.0 (October 18, 2014)
+---------------------------------
 
 {{ header }}
 
@@ -105,7 +105,7 @@ For full docs, see the :ref:`categorical introduction <categorical>` and the
 
 .. _whatsnew_0150.timedeltaindex:
 
-TimedeltaIndex/Scalar
+TimedeltaIndex/scalar
 ^^^^^^^^^^^^^^^^^^^^^
 
 We introduce a new scalar type ``Timedelta``, which is a subclass of ``datetime.timedelta``, and behaves in a similar manner,
@@ -247,8 +247,8 @@ Additionally :meth:`~pandas.DataFrame.memory_usage` is an available method for a
 
 .. _whatsnew_0150.dt:
 
-.dt accessor
-^^^^^^^^^^^^
+Series.dt accessor
+^^^^^^^^^^^^^^^^^^
 
 ``Series`` has gained an accessor to succinctly return datetime like properties for the *values* of the Series, if its a datetime/period like Series. (:issue:`7207`)
 This will return a Series, indexed like the existing Series. See the :ref:`docs <basics.dt_accessors>`
@@ -600,7 +600,7 @@ Rolling/expanding moments improvements
 
 .. _whatsnew_0150.sql:
 
-Improvements in the SQL io module
+Improvements in the SQL IO module
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 - Added support for a ``chunksize`` parameter to ``to_sql`` function. This allows DataFrame to be written in chunks and avoid packet-size overflow errors (:issue:`8062`).

doc/source/whatsnew/v0.15.1.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0151:
 
-v0.15.1 (November 9, 2014)
---------------------------
+Version 0.15.1 (November 9, 2014)
+---------------------------------
 
 {{ header }}
 

doc/source/whatsnew/v0.15.2.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0152:
 
-v0.15.2 (December 12, 2014)
----------------------------
+Version 0.15.2 (December 12, 2014)
+----------------------------------
 
 {{ header }}
 

doc/source/whatsnew/v0.16.0.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0160:
 
-v0.16.0 (March 22, 2015)
-------------------------
+Version 0.16.0 (March 22, 2015)
+-------------------------------
 
 {{ header }}
 
@@ -218,7 +218,7 @@ Backwards incompatible API changes
 
 .. _whatsnew_0160.api_breaking.timedelta:
 
-Changes in Timedelta
+Changes in timedelta
 ^^^^^^^^^^^^^^^^^^^^
 
 In v0.15.0 a new scalar type ``Timedelta`` was introduced, that is a

doc/source/whatsnew/v0.16.1.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0161:
 
-v0.16.1 (May 11, 2015)
-----------------------
+Version 0.16.1 (May 11, 2015)
+-----------------------------
 
 {{ header }}
 

doc/source/whatsnew/v0.16.2.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0162:
 
-v0.16.2 (June 12, 2015)
------------------------
+Version 0.16.2 (June 12, 2015)
+------------------------------
 
 {{ header }}
 

doc/source/whatsnew/v0.17.0.rst

@@ -1,7 +1,7 @@
 .. _whatsnew_0170:
 
-v0.17.0 (October 9, 2015)
--------------------------
+Version 0.17.0 (October 9, 2015)
+--------------------------------
 
 {{ header }}
 
@@ -181,8 +181,8 @@ Each method signature only includes relevant arguments. Currently, these are lim
 Additional methods for ``dt`` accessor
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-strftime
-""""""""
+Series.dt.strftime
+""""""""""""""""""
 
 We are now supporting a ``Series.dt.strftime`` method for datetime-likes to generate a formatted string (:issue:`10110`). Examples:
 
@@ -202,8 +202,8 @@ We are now supporting a ``Series.dt.strftime`` method for datetime-likes to gene
 
 The string format is as the python standard library and details can be found `here <https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
 
-total_seconds
-"""""""""""""
+Series.dt.total_seconds
+"""""""""""""""""""""""
 
 ``pd.Series`` of type ``timedelta64`` has new method ``.dt.total_seconds()`` returning the duration of the timedelta in seconds (:issue:`10817`)