Merge branch 'main' into revert-cache-stacklevel

Author: MarcoGorelli

asv_bench/benchmarks/multiindex_object.py

@@ -239,10 +239,11 @@ class SetOperations:
         ("monotonic", "non_monotonic"),
         ("datetime", "int", "string", "ea_int"),
         ("intersection", "union", "symmetric_difference"),
+        (False, None),
     ]
-    param_names = ["index_structure", "dtype", "method"]
+    param_names = ["index_structure", "dtype", "method", "sort"]
 
-    def setup(self, index_structure, dtype, method):
+    def setup(self, index_structure, dtype, method, sort):
         N = 10**5
         level1 = range(1000)
 
@@ -272,8 +273,8 @@ def setup(self, index_structure, dtype, method):
         self.left = data[dtype]["left"]
         self.right = data[dtype]["right"]
 
-    def time_operation(self, index_structure, dtype, method):
-        getattr(self.left, method)(self.right)
+    def time_operation(self, index_structure, dtype, method, sort):
+        getattr(self.left, method)(self.right, sort=sort)
 
 
 class Difference:

doc/redirects.csv

@@ -45,6 +45,7 @@ contributing_docstring,development/contributing_docstring
 developer,development/developer
 extending,development/extending
 internals,development/internals
+development/meeting,development/community
 
 # api moved function
 reference/api/pandas.io.json.json_normalize,pandas.json_normalize

doc/source/development/community.rst

@@ -0,0 +1,117 @@
+.. _community:
+
+=====================
+Contributor community
+=====================
+
+pandas is a community-driven open source project developed by a large group
+of `contributors <https://github.com/pandas-dev/pandas/graphs/contributors>`_
+and a smaller group of `maintainers <https://pandas.pydata.org/about/team.html>`_.
+The pandas leadership has made a strong commitment to creating an open,
+inclusive, and positive community. Please read the pandas `Code of Conduct
+<https://pandas.pydata.org/community/coc.html>`_ for guidance on how to
+interact with others in a way that makes the community thrive.
+
+We offer several meetings and communication channels to share knowledge and
+connect with others within the pandas community.
+
+Community meeting
+-----------------
+
+The pandas Community Meeting is a regular sync meeting for the project's
+maintainers which is open to the community. Everyone is welcome to attend and
+contribute to conversations.
+
+The meetings take place on the second Wednesday of each month at 18:00 UTC.
+
+The minutes of past meetings are available in `this Google Document <https://docs.google.com/document/d/1tGbTiYORHiSPgVMXawiweGJlBw5dOkVJLY-licoBmBU/edit?usp=sharing>`__.
+
+
+New contributor meeting
+-----------------------
+
+On the third Wednesday of the month, we hold meetings to welcome and support
+new contributors in our community.
+
+| 👋 you all are invited
+| 💬 everyone can present (add yourself to the hackMD agenda)
+| 👀 anyone can sit in and listen
+
+Attendees are new and experienced contributors, as well as a few maintainers.
+We aim to answer questions about getting started, or help with work in
+progress when possible, as well as get to know each other and share our
+learnings and experiences.
+
+The agenda for the next meeting and minutes of past meetings are available in
+`this HackMD <https://hackmd.io/@pandas-dev/HJgQt1Tei>`__.
+
+Calendar
+--------
+
+This calendar shows all the community meetings. Our community meetings are
+ideal for anyone wanting to contribute to pandas, or just curious to know how
+current development is going.
+
+.. raw:: html
+
+   <iframe src="https://calendar.google.com/calendar/embed?src=pgbn14p6poja8a1cf2dv2jhrmg%40group.calendar.google.com" style="border: 0" width="800" height="600" frameborder="0" scrolling="no"></iframe>
+
+You can subscribe to this calendar with the following links:
+
+* `iCal <https://calendar.google.com/calendar/ical/pgbn14p6poja8a1cf2dv2jhrmg%40group.calendar.google.com/public/basic.ics>`__
+* `Google calendar <https://calendar.google.com/calendar/[email protected]>`__
+
+Additionally, we'll sometimes have one-off meetings on specific topics.
+These will be published on the same calendar.
+
+`GitHub issue tracker <https://github.com/pandas-dev/pandas/issues>`_
+----------------------------------------------------------------------
+
+The pandas contributor community conducts conversations mainly via this channel.
+Any community member can open issues to:
+
+- Report bugs, e.g. "I noticed the behavior of a certain function is
+  incorrect"
+- Request features, e.g. "I would like this error message to be more readable"
+- Request documentation improvements, e.g. "I found this section unclear"
+- Ask questions, e.g. "I noticed the behavior of a certain function
+  changed between versions. Is this expected?".
+
+    Ideally your questions should be related to how pandas work rather
+    than how you use pandas. `StackOverflow <https://stackoverflow.com/>`_ is
+    better suited for answering usage questions, and we ask that all usage
+    questions are first asked on StackOverflow. Thank you for respecting are
+    time and wishes. 🙇
+
+Maintainers and frequent contributors might also open issues to discuss the
+ongoing development of the project. For example:
+
+- Report issues with the CI, GitHub Actions, or the performance of pandas
+- Open issues relating to the internals
+- Start roadmap discussion aligning on proposals what to do in future
+  releases or changes to the API.
+- Open issues relating to the project's website, logo, or governance
+
+The developer mailing list
+--------------------------
+
+The pandas mailing list `[email protected] <mailto://pandas-dev@python
+.org>`_ is used for long form
+conversations and to engages people in the wider community who might not
+be active on the issue tracker but we would like to include in discussions.
+
+Community slack
+---------------
+
+We have a chat platform for contributors, maintainers and potential
+contributors. This is not a space for user questions, rather for questions about
+contributing to pandas. The slack is a private space, specifically meant for
+people who are hesitant to bring up their questions or ideas on a large public
+mailing list or GitHub.
+
+If this sounds like the right place for you, you are welcome to join! Email us
+at `[email protected] <mailto://[email protected]>`_ and let us
+know that you read and agree to our `Code of Conduct <https://pandas.pydata.org/community/coc.html>`_
+😉 to get an invite. And please remember the slack is not meant to replace the
+mailing list or issue tracker - all important announcements and conversations
+should still happen there.

doc/source/development/index.rst

@@ -23,4 +23,4 @@ Development
     developer
     policies
     roadmap
-    meeting
+    community

doc/source/development/meeting.rst

@@ -10,13 +10,13 @@ This is the list of changes to pandas between each release. For full details,
 see the `commit logs <https://github.com/pandas-dev/pandas/commits/>`_. For install and
 upgrade instructions, see :ref:`install`.
 
-Version 1.6
+Version 2.0
 -----------
 
 .. toctree::
    :maxdepth: 2
 
-   v1.6.0
+   v2.0.0
 
 Version 1.5
 -----------

doc/source/whatsnew/index.rst

@@ -85,6 +85,7 @@ Fixed regressions
 - Fixed Regression in :meth:`DataFrameGroupBy.apply` when user defined function is called on an empty dataframe (:issue:`47985`)
 - Fixed regression in :meth:`DataFrame.apply` when passing non-zero ``axis`` via keyword argument (:issue:`48656`)
 - Fixed regression in :meth:`Series.groupby` and :meth:`DataFrame.groupby` when the grouper is a nullable data type (e.g. :class:`Int64`) or a PyArrow-backed string array, contains null values, and ``dropna=False`` (:issue:`48794`)
+- Fixed regression in :meth:`DataFrame.to_parquet` raising when file name was specified as ``bytes`` (:issue:`48944`)
 - Fixed regression in :class:`ExcelWriter` where the ``book`` attribute could no longer be set; however setting this attribute is now deprecated and this ability will be removed in a future version of pandas (:issue:`48780`)
 - Fixed regression causing memory leak when finding stacklevel when emitting warnings (:issue:`49052`)
 

doc/source/whatsnew/v1.5.1.rst

@@ -1,30 +1,30 @@
-.. _whatsnew_160:
+.. _whatsnew_200:
 
-What's new in 1.6.0 (??)
+What's new in 2.0.0 (??)
 ------------------------
 
-These are the changes in pandas 1.6.0. See :ref:`release` for a full changelog
+These are the changes in pandas 2.0.0. See :ref:`release` for a full changelog
 including other versions of pandas.
 
 {{ header }}
 
 .. ---------------------------------------------------------------------------
-.. _whatsnew_160.enhancements:
+.. _whatsnew_200.enhancements:
 
 Enhancements
 ~~~~~~~~~~~~
 
-.. _whatsnew_160.enhancements.enhancement1:
+.. _whatsnew_200.enhancements.enhancement1:
 
 enhancement1
 ^^^^^^^^^^^^
 
-.. _whatsnew_160.enhancements.enhancement2:
+.. _whatsnew_200.enhancements.enhancement2:
 
 enhancement2
 ^^^^^^^^^^^^
 
-.. _whatsnew_160.enhancements.other:
+.. _whatsnew_200.enhancements.other:
 
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
@@ -39,14 +39,14 @@ Other enhancements
 -
 
 .. ---------------------------------------------------------------------------
-.. _whatsnew_160.notable_bug_fixes:
+.. _whatsnew_200.notable_bug_fixes:
 
 Notable bug fixes
 ~~~~~~~~~~~~~~~~~
 
 These are bug fixes that might have notable behavior changes.
 
-.. _whatsnew_160.notable_bug_fixes.notable_bug_fix1:
+.. _whatsnew_200.notable_bug_fixes.notable_bug_fix1:
 
 :meth:`.GroupBy.cumsum` and :meth:`.GroupBy.cumprod` overflow instead of lossy casting to float
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -76,18 +76,18 @@ We return incorrect results with the 6th value.
 
 We overflow with the 7th value, but the 6th value is still correct.
 
-.. _whatsnew_160.notable_bug_fixes.notable_bug_fix2:
+.. _whatsnew_200.notable_bug_fixes.notable_bug_fix2:
 
 notable_bug_fix2
 ^^^^^^^^^^^^^^^^
 
 .. ---------------------------------------------------------------------------
-.. _whatsnew_160.api_breaking:
+.. _whatsnew_200.api_breaking:
 
 Backwards incompatible API changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. _whatsnew_160.api_breaking.deps:
+.. _whatsnew_200.api_breaking.deps:
 
 Increased minimum versions for dependencies
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -112,7 +112,7 @@ Optional libraries below the lowest tested version may still work, but are not c
 
 See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more.
 
-.. _whatsnew_160.api_breaking.other:
+.. _whatsnew_200.api_breaking.other:
 
 Other API changes
 ^^^^^^^^^^^^^^^^^
@@ -121,18 +121,19 @@ Other API changes
 - Default value of ``dtype`` in :func:`get_dummies` is changed to ``bool`` from ``uint8`` (:issue:`45848`)
 - :meth:`DataFrame.astype`, :meth:`Series.astype`, and :meth:`DatetimeIndex.astype` casting datetime64 data to any of "datetime64[s]", "datetime64[ms]", "datetime64[us]" will return an object with the given resolution instead of coercing back to "datetime64[ns]" (:issue:`48928`)
 - :meth:`DataFrame.astype`, :meth:`Series.astype`, and :meth:`DatetimeIndex.astype` casting timedelta64 data to any of "timedelta64[s]", "timedelta64[ms]", "timedelta64[us]" will return an object with the given resolution instead of coercing to "float64" dtype (:issue:`48963`)
+- Passing a ``np.datetime64`` object with non-nanosecond resolution to :class:`Timestamp` will retain the input resolution if it is "s", "ms", or "ns"; otherwise it will be cast to the closest supported resolution (:issue:`49008`)
 -
 
 .. ---------------------------------------------------------------------------
-.. _whatsnew_160.deprecations:
+.. _whatsnew_200.deprecations:
 
 Deprecations
 ~~~~~~~~~~~~
 -
 -
 
 .. ---------------------------------------------------------------------------
-.. _whatsnew_160.performance:
+.. _whatsnew_200.performance:
 
 Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
@@ -141,6 +142,7 @@ Performance improvements
 - Performance improvement in :meth:`MultiIndex.size` (:issue:`48723`)
 - Performance improvement in :meth:`MultiIndex.union` without missing values and without duplicates (:issue:`48505`)
 - Performance improvement in :meth:`MultiIndex.difference` (:issue:`48606`)
+- Performance improvement in :class:`MultiIndex` set operations with sort=None (:issue:`49010`)
 - Performance improvement in :meth:`.DataFrameGroupBy.mean`, :meth:`.SeriesGroupBy.mean`, :meth:`.DataFrameGroupBy.var`, and :meth:`.SeriesGroupBy.var` for extension array dtypes (:issue:`37493`)
 - Performance improvement in :meth:`MultiIndex.isin` when ``level=None`` (:issue:`48622`)
 - Performance improvement in :meth:`Index.union` and :meth:`MultiIndex.union` when index contains duplicates (:issue:`48900`)
@@ -158,7 +160,7 @@ Performance improvements
 - Performance improvement in :class:`DataFrameGroupBy` and :class:`SeriesGroupBy` when ``by`` is a categorical type and ``sort=False`` (:issue:`48976`)
 
 .. ---------------------------------------------------------------------------
-.. _whatsnew_160.bug_fixes:
+.. _whatsnew_200.bug_fixes:
 
 Bug fixes
 ~~~~~~~~~
@@ -230,6 +232,7 @@ MultiIndex
 - Bug in :meth:`MultiIndex.unique` losing extension array dtype (:issue:`48335`)
 - Bug in :meth:`MultiIndex.intersection` losing extension array (:issue:`48604`)
 - Bug in :meth:`MultiIndex.union` losing extension array (:issue:`48498`, :issue:`48505`, :issue:`48900`)
+- Bug in :meth:`MultiIndex.union` not sorting when sort=None and index contains missing values (:issue:`49010`)
 - Bug in :meth:`MultiIndex.append` not checking names for equality (:issue:`48288`)
 - Bug in :meth:`MultiIndex.symmetric_difference` losing extension array (:issue:`48607`)
 -
@@ -294,7 +297,7 @@ Other
 -
 
 .. ---------------------------------------------------------------------------
-.. _whatsnew_160.contributors:
+.. _whatsnew_200.contributors:
 
 Contributors
 ~~~~~~~~~~~~

doc/source/whatsnew/v1.6.0.rst renamed to doc/source/whatsnew/v2.0.0.rst

@@ -513,6 +513,7 @@ cpdef array_to_datetime(
                         found_tz = True
                         if utc_convert:
                             _ts = convert_datetime_to_tsobject(val, None)
+                            _ts.ensure_reso(NPY_FR_ns)
                             iresult[i] = _ts.value
                         elif found_naive:
                             raise ValueError('Tz-aware datetime.datetime '
@@ -526,6 +527,7 @@ cpdef array_to_datetime(
                             found_tz = True
                             tz_out = val.tzinfo
                             _ts = convert_datetime_to_tsobject(val, None)
+                            _ts.ensure_reso(NPY_FR_ns)
                             iresult[i] = _ts.value
 
                     else:
@@ -534,7 +536,7 @@ cpdef array_to_datetime(
                             raise ValueError('Cannot mix tz-aware with '
                                              'tz-naive values')
                         if isinstance(val, _Timestamp):
-                            iresult[i] = val.value
+                            iresult[i] = val._as_unit("ns").value
                         else:
                             iresult[i] = pydatetime_to_dt64(val, &dts)
                             check_dts_bounds(&dts)

pandas/_libs/tslib.pyx

@@ -20,6 +20,9 @@ cdef class _TSObject:
         int64_t value               # numpy dt64
         tzinfo tzinfo
         bint fold
+        NPY_DATETIMEUNIT reso
+
+    cdef void ensure_reso(self, NPY_DATETIMEUNIT reso)
 
 
 cdef _TSObject convert_to_tsobject(object ts, tzinfo tz, str unit,