Merge branch 'master' into parquet-categorical

Author: galuhsahid

.gitignore

@@ -57,6 +57,7 @@ dist
 # wheel files
 *.whl
 **/wheelhouse/*
+pip-wheel-metadata
 # coverage
 .coverage
 coverage.xml

.pre-commit-config.yaml

@@ -1,17 +1,18 @@
 repos:
-    - repo: https://github.com/python/black
-      rev: stable
-      hooks:
-          - id: black
-            language_version: python3.7
-    - repo: https://gitlab.com/pycqa/flake8
-      rev: 3.7.7
-      hooks:
-          - id: flake8
-            language: python_venv
-            additional_dependencies: [flake8-comprehensions]
-    - repo: https://github.com/pre-commit/mirrors-isort
-      rev: v4.3.20
-      hooks:
-          - id: isort
-            language: python_venv
+-   repo: https://github.com/python/black
+    rev: stable
+    hooks:
+    -   id: black
+        language_version: python3.7
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.7.7
+    hooks:
+    -   id: flake8
+        language: python_venv
+        additional_dependencies: [flake8-comprehensions]
+-   repo: https://github.com/pre-commit/mirrors-isort
+    rev: v4.3.20
+    hooks:
+    -   id: isort
+        language: python_venv
+        exclude: ^pandas/__init__\.py$|^pandas/core/api\.py$

MANIFEST.in

@@ -3,6 +3,7 @@ include LICENSE
 include RELEASE.md
 include README.md
 include setup.py
+include pyproject.toml
 
 graft doc
 prune doc/build
@@ -14,6 +15,7 @@ graft pandas
 global-exclude *.bz2
 global-exclude *.csv
 global-exclude *.dta
+global-exclude *.feather
 global-exclude *.gz
 global-exclude *.h5
 global-exclude *.html
@@ -23,7 +25,10 @@ global-exclude *.pickle
 global-exclude *.png
 global-exclude *.pyc
 global-exclude *.pyd
+global-exclude *.ods
+global-exclude *.odt
 global-exclude *.sas7bdat
+global-exclude *.sav
 global-exclude *.so
 global-exclude *.xls
 global-exclude *.xlsm

Makefile

@@ -18,7 +18,7 @@ black:
 	black . --exclude '(asv_bench/env|\.egg|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|_build|buck-out|build|dist|setup.py)'
 
 develop: build
-	python setup.py develop
+	python -m pip install --no-build-isolation -e .
 
 doc:
 	-rm -rf doc/build doc/source/generated

README.md

@@ -188,16 +188,17 @@ python setup.py install
 
 or for installing in [development mode](https://pip.pypa.io/en/latest/reference/pip_install.html#editable-installs):
 
+
 ```sh
-python setup.py develop
+python -m pip install --no-build-isolation -e .
 ```
 
-Alternatively, you can use `pip` if you want all the dependencies pulled
-in automatically (the `-e` option is for installing it in [development
-mode](https://pip.pypa.io/en/latest/reference/pip_install.html#editable-installs)):
+If you have `make`, you can also use `make develop` to run the same command.
+
+or alternatively
 
 ```sh
-pip install -e .
+python setup.py develop
 ```
 
 See the full instructions for [installing from source](https://pandas.pydata.org/pandas-docs/stable/install.html#installing-from-source).
@@ -224,7 +225,7 @@ Most development discussion is taking place on github in this repo. Further, the
 
 All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome.
 
-A detailed overview on how to contribute can be found in the **[contributing guide](https://dev.pandas.io/contributing.html)**. There is also an [overview](.github/CONTRIBUTING.md) on GitHub.
+A detailed overview on how to contribute can be found in the **[contributing guide](https://dev.pandas.io/docs/contributing.html)**. There is also an [overview](.github/CONTRIBUTING.md) on GitHub.
 
 If you are simply looking to start working with the pandas codebase, navigate to the [GitHub "issues" tab](https://github.com/pandas-dev/pandas/issues) and start looking through interesting issues. There are a number of issues listed under [Docs](https://github.com/pandas-dev/pandas/issues?labels=Docs&sort=updated&state=open) and [good first issue](https://github.com/pandas-dev/pandas/issues?labels=good+first+issue&sort=updated&state=open) where you could start out.
 

asv_bench/asv.conf.json

@@ -50,12 +50,13 @@
         "xlsxwriter": [],
         "xlrd": [],
         "xlwt": [],
+        "odfpy": [],
         "pytest": [],
         // If using Windows with python 2.7 and want to build using the
         // mingw toolchain (rather than MSVC), uncomment the following line.
         // "libpython": [],
     },
-
+    "conda_channels": ["defaults", "conda-forge"],
     // Combinations of libraries/python versions can be excluded/included
     // from the set to test. Each entry is a dictionary containing additional
     // key-value pairs to include/exclude.

asv_bench/benchmarks/attrs_caching.py

@@ -1,4 +1,5 @@
 import numpy as np
+
 from pandas import DataFrame
 
 try:
@@ -32,4 +33,4 @@ def time_cache_readonly(self):
         self.obj.prop
 
 
-from .pandas_vb_common import setup  # noqa: F401
+from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/binary_ops.py

@@ -1,4 +1,5 @@
 import numpy as np
+
 from pandas import DataFrame, Series, date_range
 from pandas.core.algorithms import checked_add_with_arr
 
@@ -155,4 +156,4 @@ def time_add_overflow_both_arg_nan(self):
         )
 
 
-from .pandas_vb_common import setup  # noqa: F401
+from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/categoricals.py

@@ -1,7 +1,9 @@
+import warnings
+
 import numpy as np
+
 import pandas as pd
 import pandas.util.testing as tm
-import warnings
 
 try:
     from pandas.api.types import union_categoricals
@@ -280,4 +282,4 @@ def time_sort_values(self):
         self.index.sort_values(ascending=False)
 
 
-from .pandas_vb_common import setup  # noqa: F401
+from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/ctors.py

@@ -1,6 +1,7 @@
 import numpy as np
+
+from pandas import DatetimeIndex, Index, MultiIndex, Series, Timestamp
 import pandas.util.testing as tm
-from pandas import Series, Index, DatetimeIndex, Timestamp, MultiIndex
 
 
 def no_change(arr):
@@ -113,4 +114,4 @@ def time_multiindex_from_iterables(self):
         MultiIndex.from_product(self.iterables)
 
 
-from .pandas_vb_common import setup  # noqa: F401
+from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/dtypes.py

@@ -1,14 +1,14 @@
+import numpy as np
+
 from pandas.api.types import pandas_dtype
 
-import numpy as np
 from .pandas_vb_common import (
-    numeric_dtypes,
     datetime_dtypes,
-    string_dtypes,
     extension_dtypes,
+    numeric_dtypes,
+    string_dtypes,
 )
 
-
 _numpy_dtypes = [
     np.dtype(dtype) for dtype in (numeric_dtypes + datetime_dtypes + string_dtypes)
 ]
@@ -40,4 +40,4 @@ def time_pandas_dtype_invalid(self, dtype):
             pass
 
 
-from .pandas_vb_common import setup  # noqa: F401
+from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/eval.py

@@ -1,4 +1,5 @@
 import numpy as np
+
 import pandas as pd
 
 try:
@@ -62,4 +63,4 @@ def time_query_with_boolean_selection(self):
         self.df.query("(a >= @self.min_val) & (a <= @self.max_val)")
 
 
-from .pandas_vb_common import setup  # noqa: F401
+from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/frame_ctor.py

@@ -1,6 +1,7 @@
 import numpy as np
+
+from pandas import DataFrame, MultiIndex, Series, Timestamp, date_range
 import pandas.util.testing as tm
-from pandas import DataFrame, Series, MultiIndex, Timestamp, date_range
 
 try:
     from pandas.tseries.offsets import Nano, Hour
@@ -104,4 +105,4 @@ def time_frame_from_lists(self):
         self.df = DataFrame(self.data)
 
 
-from .pandas_vb_common import setup  # noqa: F401
+from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/frame_methods.py

@@ -1,5 +1,5 @@
-import warnings
 import string
+import warnings
 
 import numpy as np
 
@@ -609,4 +609,15 @@ def time_dataframe_describe(self):
         self.df.describe()
 
 
-from .pandas_vb_common import setup  # noqa: F401
+class SelectDtypes:
+    params = [100, 1000]
+    param_names = ["n"]
+
+    def setup(self, n):
+        self.df = DataFrame(np.random.randn(10, n))
+
+    def time_select_dtypes(self, n):
+        self.df.select_dtypes(include="int")
+
+
+from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/gil.py

@@ -1,7 +1,8 @@
 import numpy as np
-import pandas.util.testing as tm
-from pandas import DataFrame, Series, read_csv, factorize, date_range
+
+from pandas import DataFrame, Series, date_range, factorize, read_csv
 from pandas.core.algorithms import take_1d
+import pandas.util.testing as tm
 
 try:
     from pandas import (
@@ -36,7 +37,7 @@ def wrapper(fname):
         return wrapper
 
 
-from .pandas_vb_common import BaseIO
+from .pandas_vb_common import BaseIO  # noqa: E402 isort:skip
 
 
 class ParallelGroupbyMethods:
@@ -301,4 +302,4 @@ def time_loop(self, threads):
             self.loop()
 
 
-from .pandas_vb_common import setup  # noqa: F401
+from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/groupby.py

@@ -15,7 +15,6 @@
 )
 import pandas.util.testing as tm
 
-
 method_blacklist = {
     "object": {
         "median",
@@ -626,4 +625,4 @@ def time_first(self):
         self.df_nans.groupby("key").transform("first")
 
 
-from .pandas_vb_common import setup  # noqa: F401
+from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/index_object.py

@@ -1,15 +1,17 @@
 import gc
+
 import numpy as np
-import pandas.util.testing as tm
+
 from pandas import (
-    Series,
-    date_range,
     DatetimeIndex,
-    Index,
-    RangeIndex,
     Float64Index,
+    Index,
     IntervalIndex,
+    RangeIndex,
+    Series,
+    date_range,
 )
+import pandas.util.testing as tm
 
 
 class SetOperations:
@@ -243,4 +245,4 @@ def peakmem_gc_instances(self, N):
             gc.enable()
 
 
-from .pandas_vb_common import setup  # noqa: F401
+from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/indexing.py

@@ -1,22 +1,23 @@
 import warnings
 
 import numpy as np
-import pandas.util.testing as tm
+
 from pandas import (
-    Series,
+    CategoricalIndex,
     DataFrame,
-    MultiIndex,
-    Int64Index,
-    UInt64Index,
     Float64Index,
-    IntervalIndex,
-    CategoricalIndex,
     IndexSlice,
+    Int64Index,
+    IntervalIndex,
+    MultiIndex,
+    Series,
+    UInt64Index,
     concat,
     date_range,
     option_context,
     period_range,
 )
+import pandas.util.testing as tm
 
 
 class NumericSeriesIndexing:
@@ -371,4 +372,4 @@ def time_chained_indexing(self, mode):
                 df2["C"] = 1.0
 
 
-from .pandas_vb_common import setup  # noqa: F401
+from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/inference.py

@@ -1,8 +1,9 @@
 import numpy as np
-import pandas.util.testing as tm
+
 from pandas import DataFrame, Series, to_numeric
+import pandas.util.testing as tm
 
-from .pandas_vb_common import numeric_dtypes, lib
+from .pandas_vb_common import lib, numeric_dtypes
 
 
 class NumericInferOps:
@@ -120,4 +121,4 @@ def time_convert(self, data):
         lib.maybe_convert_numeric(data, set(), coerce_numeric=False)
 
 
-from .pandas_vb_common import setup  # noqa: F401
+from .pandas_vb_common import setup  # noqa: F401 isort:skip