merge for update

Author: makbigc

asv_bench/benchmarks/frame_methods.py

@@ -609,4 +609,15 @@ def time_dataframe_describe(self):
         self.df.describe()
 
 
+class SelectDtypes:
+    params = [100, 1000]
+    param_names = ["n"]
+
+    def setup(self, n):
+        self.df = DataFrame(np.random.randn(10, n))
+
+    def time_select_dtypes(self, n):
+        self.df.select_dtypes(include="int")
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/rolling.py

@@ -25,6 +25,25 @@ def peakmem_rolling(self, constructor, window, dtype, method):
         getattr(self.roll, method)()
 
 
+class Apply:
+    params = (
+        ["DataFrame", "Series"],
+        [10, 1000],
+        ["int", "float"],
+        [sum, np.sum, lambda x: np.sum(x) + 5],
+        [True, False],
+    )
+    param_names = ["contructor", "window", "dtype", "function", "raw"]
+
+    def setup(self, constructor, window, dtype, function, raw):
+        N = 10 ** 5
+        arr = (100 * np.random.random(N)).astype(dtype)
+        self.roll = getattr(pd, constructor)(arr).rolling(window)
+
+    def time_rolling(self, constructor, window, dtype, function, raw):
+        self.roll.apply(function, raw=raw)
+
+
 class ExpandingMethods:
 
     params = (

azure-pipelines.yml

@@ -121,7 +121,7 @@ jobs:
 
   - script: |
       source activate pandas-dev
-      python web/pandas_web.py web/pandas
+      python web/pandas_web.py web/pandas --target-path=web/build
     displayName: 'Build website'
 
   - script: |
@@ -171,7 +171,7 @@ jobs:
           eq(variables['Build.SourceBranch'], 'refs/heads/master'))
 
   - script: |
-      cd doc/build/html
+      cd to_deploy
       git remote add origin [email protected]:pandas-dev/pandas-dev.github.io.git
       git push -f origin master
     displayName: 'Publish web and docs to GitHub pages'

ci/azure/posix.yml

@@ -60,15 +60,21 @@ jobs:
         echo "Creating Environment"
         ci/setup_env.sh
       displayName: 'Setup environment and build pandas'
+
     - script: |
         source activate pandas-dev
         ci/run_tests.sh
       displayName: 'Test'
+
     - script: source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
+      displayName: 'Build versions'
+
     - task: PublishTestResults@2
       inputs:
         testResultsFiles: 'test-data-*.xml'
         testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }}
+      displayName: 'Publish test results'
+
     - powershell: |
         $junitXml = "test-data-single.xml"
         $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
@@ -94,6 +100,7 @@ jobs:
           Write-Error "$($matches[1]) tests failed"
         }
       displayName: 'Check for test failures'
+
     - script: |
         source activate pandas-dev
         python ci/print_skipped.py

ci/code_checks.sh

@@ -125,6 +125,10 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     # invgrep -R --include="*.py*" -E "from numpy import nan " pandas  # GH#24822 not yet implemented since the offending imports have not all been removed
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Check for use of exec' ; echo $MSG
+    invgrep -R --include="*.py*" -E "[^a-zA-Z0-9_]exec\(" pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     MSG='Check for pytest warns' ; echo $MSG
     invgrep -r -E --include '*.py' 'pytest\.warns' pandas/tests/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
@@ -184,7 +188,7 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -R --include="*.rst" ".. ipython ::" doc/source
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    MSG='Check that no file in the repo contains tailing whitespaces' ; echo $MSG
+    MSG='Check that no file in the repo contains trailing whitespaces' ; echo $MSG
     set -o pipefail
     if [[ "$AZURE" == "true" ]]; then
         # we exclude all c/cpp files as the c/cpp files of pandas code base are tested when Linting .c and .h files
@@ -262,6 +266,10 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
         -k"-from_arrays -from_breaks -from_intervals -from_tuples -set_closed -to_tuples -interval_range"
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Doctests arrays/string_.py' ; echo $MSG
+    pytest -q --doctest-modules pandas/core/arrays/string_.py
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
 fi
 
 ### DOCSTRINGS ###

ci/deps/travis-36-cov.yaml

@@ -29,7 +29,7 @@ dependencies:
   - python-snappy
   - python=3.6.*
   - pytz
-  - s3fs
+  - s3fs<0.3
   - scikit-learn
   - scipy
   - sqlalchemy

ci/deps/travis-36-slow.yaml

@@ -18,7 +18,7 @@ dependencies:
   - python-dateutil
   - python=3.6.*
   - pytz
-  - s3fs
+  - s3fs<0.3
   - scipy
   - sqlalchemy
   - xlrd

ci/deps/travis-37.yaml

@@ -17,7 +17,7 @@ dependencies:
   - pytest-xdist>=1.29.0
   - pytest-mock
   - hypothesis>=3.58.0
-  - s3fs
+  - s3fs<0.3
   - pip
   - pyreadstat
   - pip:

ci/print_skipped.py

@@ -1,52 +1,40 @@
 #!/usr/bin/env python
-
-import math
 import os
-import sys
 import xml.etree.ElementTree as et
 
 
-def parse_results(filename):
+def main(filename):
+    if not os.path.isfile(filename):
+        return
+
     tree = et.parse(filename)
     root = tree.getroot()
-    skipped = []
-
     current_class = ""
-    i = 1
-    assert i - 1 == len(skipped)
     for el in root.findall("testcase"):
         cn = el.attrib["classname"]
         for sk in el.findall("skipped"):
             old_class = current_class
             current_class = cn
-            name = "{classname}.{name}".format(
-                classname=current_class, name=el.attrib["name"]
-            )
-            msg = sk.attrib["message"]
-            out = ""
             if old_class != current_class:
-                ndigits = int(math.log(i, 10) + 1)
-
-                # 4 for : + space + # + space
-                out += "-" * (len(name + msg) + 4 + ndigits) + "\n"
-            out += "#{i} {name}: {msg}".format(i=i, name=name, msg=msg)
-            skipped.append(out)
-            i += 1
-            assert i - 1 == len(skipped)
-    assert i - 1 == len(skipped)
-    # assert len(skipped) == int(root.attrib['skip'])
-    return "\n".join(skipped)
-
-
-def main():
-    test_files = ["test-data-single.xml", "test-data-multiple.xml", "test-data.xml"]
-
-    print("SKIPPED TESTS:")
-    for fn in test_files:
-        if os.path.isfile(fn):
-            print(parse_results(fn))
-    return 0
+                yield None
+            yield {
+                "class_name": current_class,
+                "test_name": el.attrib["name"],
+                "message": sk.attrib["message"],
+            }
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    print("SKIPPED TESTS:")
+    i = 1
+    for file_type in ("-single", "-multiple", ""):
+        for test_data in main("test-data{}.xml".format(file_type)):
+            if test_data is None:
+                print("-" * 80)
+            else:
+                print(
+                    "#{i} {class_name}.{test_name}: {message}".format(
+                        **dict(test_data, i=i)
+                    )
+                )
+                i += 1

ci/run_tests.sh

@@ -1,13 +1,6 @@
-#!/bin/bash
+#!/bin/bash -e
 
-set -e
-
-if [ "$DOC" ]; then
-    echo "We are not running pytest as this is a doc-build"
-    exit 0
-fi
-
-# Workaround for pytest-xdist flaky collection order
+# Workaround for pytest-xdist (it collects different tests in the workers if PYTHONHASHSEED is not set)
 # https://github.com/pytest-dev/pytest/issues/920
 # https://github.com/pytest-dev/pytest/issues/1075
 export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))')
@@ -16,7 +9,7 @@ if [ -n "$LOCALE_OVERRIDE" ]; then
     export LC_ALL="$LOCALE_OVERRIDE"
     export LANG="$LOCALE_OVERRIDE"
     PANDAS_LOCALE=`python -c 'import pandas; pandas.get_option("display.encoding")'`
-    if [[ "$LOCALE_OVERIDE" != "$PANDAS_LOCALE" ]]; then
+    if [[ "$LOCALE_OVERRIDE" != "$PANDAS_LOCALE" ]]; then
         echo "pandas could not detect the locale. System locale: $LOCALE_OVERRIDE, pandas detected: $PANDAS_LOCALE"
         # TODO Not really aborting the tests until https://github.com/pandas-dev/pandas/issues/23923 is fixed
         # exit 1
@@ -50,10 +43,9 @@ do
     # if no tests are found (the case of "single and slow"), pytest exits with code 5, and would make the script fail, if not for the below code
     sh -c "$PYTEST_CMD; ret=\$?; [ \$ret = 5 ] && exit 0 || exit \$ret"
 
-    # 2019-08-21 disabling because this is hitting HTTP 400 errors GH#27602
-    # if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then
-    #    echo "uploading coverage for $TYPE tests"
-    #    echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME"
-    #          bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME
-    # fi
+    if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then
+        echo "uploading coverage for $TYPE tests"
+        echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME"
+              bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME
+    fi
 done

doc/.gitignore

@@ -0,0 +1,4 @@
+data/
+timeseries.csv
+timeseries.parquet
+timeseries_wide.parquet

doc/redirects.csv

@@ -6,6 +6,7 @@ whatsnew,whatsnew/index
 release,whatsnew/index
 
 # getting started
+install,getting_started/install
 10min,getting_started/10min
 basics,getting_started/basics
 comparison_with_r,getting_started/comparison/comparison_with_r
@@ -1577,3 +1578,6 @@ generated/pandas.unique,../reference/api/pandas.unique
 generated/pandas.util.hash_array,../reference/api/pandas.util.hash_array
 generated/pandas.util.hash_pandas_object,../reference/api/pandas.util.hash_pandas_object
 generated/pandas.wide_to_long,../reference/api/pandas.wide_to_long
+
+# Cached searches
+reference/api/pandas.DataFrame.from_csv,pandas.read_csv

doc/source/conf.py

@@ -191,7 +191,7 @@
 
 # The theme to use for HTML and HTML Help pages.  Major themes that come with
 # Sphinx are currently 'default' and 'sphinxdoc'.
-html_theme = "nature_with_gtoc"
+html_theme = "pandas_sphinx_theme"
 
 # The style sheet to use for HTML and HTML Help pages. A file of that name
 # must exist either in Sphinx' static/ path, or in one of the custom paths
@@ -204,7 +204,7 @@
 # html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-html_theme_path = ["themes"]
+# html_theme_path = ["themes"]
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".

doc/source/development/contributing.rst

@@ -952,7 +952,7 @@ the expected correct result::
 Transitioning to ``pytest``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-*pandas* existing test structure is *mostly* classed based, meaning that you will typically find tests wrapped in a class.
+*pandas* existing test structure is *mostly* class-based, meaning that you will typically find tests wrapped in a class.
 
 .. code-block:: python
 

doc/source/development/index.rst

@@ -16,4 +16,5 @@ Development
     internals
     extending
     developer
+    policies
     roadmap

doc/source/development/policies.rst

@@ -0,0 +1,57 @@
+.. _develop.policies:
+
+********
+Policies
+********
+
+.. _policies.version:
+
+Version Policy
+~~~~~~~~~~~~~~
+
+.. versionchanged:: 1.0.0
+
+Pandas uses a loose variant of semantic versioning (`SemVer`_) to govern
+deprecations, API compatibility, and version numbering.
+
+A pandas release number is made up of ``MAJOR.MINOR.PATCH``.
+
+API breaking changes should only occur in **major** releases. Theses changes
+will be documented, with clear guidance on what is changing, why it's changing,
+and how to migrate existing code to the new behavior.
+
+Whenever possible, a deprecation path will be provided rather than an outright
+breaking change.
+
+Pandas will introduce deprecations in **minor** releases. These deprecations
+will preserve the existing behavior while emitting a warning that provide
+guidance on:
+
+* How to achieve similar behavior if an alternative is available
+* The pandas version in which the deprecation will be enforced.
+
+We will not introduce new deprecations in patch releases.
+
+Deprecations will only be enforced in **major** releases. For example, if a
+behavior is deprecated in pandas 1.2.0, it will continue to work, with a
+warning, for all releases in the 1.x series. The behavior will change and the
+deprecation removed in the next next major release (2.0.0).
+
+.. note::
+
+   Pandas will sometimes make *behavior changing* bug fixes, as part of
+   minor or patch releases. Whether or not a change is a bug fix or an
+   API-breaking change is a judgement call. We'll do our best, and we
+   invite you to participate in development discussion on the issue
+   tracker or mailing list.
+
+These policies do not apply to features marked as **experimental** in the documentation.
+Pandas may change the behavior of experimental features at any time.
+
+Python Support
+~~~~~~~~~~~~~~
+
+Pandas will only drop support for specific Python versions (e.g. 3.5.x, 3.6.x) in
+pandas **major** releases.
+
+.. _SemVer: https://semver.org

doc/source/ecosystem.rst

@@ -1,3 +1,5 @@
+:orphan:
+
 .. _ecosystem:
 
 {{ header }}