Merge remote-tracking branch 'upstream/master' into isort-io

* upstream/master:
  Make DTI[tz]._values and Series[tz]._values return DTA (pandas-dev#24534)
  CLN: Refactor some sorting code in Index set operations (pandas-dev#24533)
  Run isort (pandas-dev#24530)
  CI: fix db usage in CI (pandas-dev#24529)

Author: thoo

.travis.yml

@@ -34,23 +34,23 @@ matrix:
     include:
     - dist: trusty
       env:
-        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="not slow and not network"
+        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)"
 
     - dist: trusty
       env:
-        - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" PATTERN="not slow and db"
+        - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" PATTERN="(not slow or (single and db))"
       addons:
         apt:
           packages:
           - python-gtk2
 
     - dist: trusty
       env:
-        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="not slow and not network and db" LOCALE_OVERRIDE="zh_CN.UTF-8"
+        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8"
 
     - dist: trusty
       env:
-        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36.yaml" PATTERN="not slow and not network and db" PANDAS_TESTING_MODE="deprecate" COVERAGE=true
+        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true
 
     # In allow_failures
     - dist: trusty

pandas/conftest.py

@@ -1,4 +1,3 @@
-import collections
 from datetime import date, time, timedelta
 from decimal import Decimal
 import importlib
@@ -55,24 +54,14 @@ def pytest_runtest_setup(item):
     if 'network' in item.keywords and item.config.getoption("--skip-network"):
         pytest.skip("skipping due to --skip-network")
 
+    if 'db' in item.keywords and item.config.getoption("--skip-db"):
+        pytest.skip("skipping due to --skip-db")
+
     if 'high_memory' in item.keywords and not item.config.getoption(
             "--run-high-memory"):
         pytest.skip(
             "skipping high memory test since --run-high-memory was not set")
 
-    # if "db" not explicitly set in the -m pattern, we skip the db tests
-    pattern = item.config.getoption('-m')
-    if 'db' in item.keywords and not pattern:
-        pytest.skip('skipping db unless -m "db" is specified')
-    elif 'db' in item.keywords and pattern:
-        markers = collections.defaultdict(bool)
-        for marker in item.iter_markers():
-            markers[marker.name] = True
-        markers['db'] = False
-        db_in_pattern = not eval(pattern, {}, markers)
-        if not db_in_pattern:
-            pytest.skip('skipping db unless -m "db" is specified')
-
 
 # Configurations for all tests and all test modules
 

pandas/core/dtypes/concat.py

@@ -426,8 +426,7 @@ def _concat_datetime(to_concat, axis=0, typs=None):
     if any(typ.startswith('datetime') for typ in typs):
 
         if 'datetime' in typs:
-            to_concat = [np.array(x, copy=False).view(np.int64)
-                         for x in to_concat]
+            to_concat = [x.astype(np.int64, copy=False) for x in to_concat]
             return _concatenate_2d(to_concat, axis=axis).view(_NS_DTYPE)
         else:
             # when to_concat has different tz, len(typs) > 1.
@@ -451,7 +450,7 @@ def _convert_datetimelike_to_object(x):
     # if dtype is of datetimetz or timezone
     if x.dtype.kind == _NS_DTYPE.kind:
         if getattr(x, 'tz', None) is not None:
-            x = x.astype(object).values
+            x = np.asarray(x.astype(object))
         else:
             shape = x.shape
             x = tslib.ints_to_pydatetime(x.view(np.int64).ravel(),

pandas/core/indexes/base.py

@@ -2302,27 +2302,15 @@ def union(self, other):
                                            allow_fill=False)
                 result = _concat._concat_compat((lvals, other_diff))
 
-                try:
-                    lvals[0] < other_diff[0]
-                except TypeError as e:
-                    warnings.warn("%s, sort order is undefined for "
-                                  "incomparable objects" % e, RuntimeWarning,
-                                  stacklevel=3)
-                else:
-                    types = frozenset((self.inferred_type,
-                                       other.inferred_type))
-                    if not types & _unsortable_types:
-                        result.sort()
-
             else:
                 result = lvals
 
-                try:
-                    result = np.sort(result)
-                except TypeError as e:
-                    warnings.warn("%s, sort order is undefined for "
-                                  "incomparable objects" % e, RuntimeWarning,
-                                  stacklevel=3)
+            try:
+                result = sorting.safe_sort(result)
+            except TypeError as e:
+                warnings.warn("%s, sort order is undefined for "
+                              "incomparable objects" % e, RuntimeWarning,
+                              stacklevel=3)
 
         # for subclasses
         return self._wrap_setop_result(other, result)

pandas/core/indexes/datetimes.py

@@ -316,6 +316,12 @@ def _simple_new(cls, values, name=None, freq=None, tz=None, dtype=None):
         we require the we have a dtype compat for the values
         if we are passed a non-dtype compat, then coerce using the constructor
         """
+        if isinstance(values, DatetimeArray):
+            values = DatetimeArray(values, freq=freq, tz=tz, dtype=dtype)
+            tz = values.tz
+            freq = values.freq
+            values = values._data
+
         # DatetimeArray._simple_new will accept either i8 or M8[ns] dtypes
         assert isinstance(values, np.ndarray), type(values)
 
@@ -340,7 +346,7 @@ def _values(self):
         # tz-naive -> ndarray
         # tz-aware -> DatetimeIndex
         if self.tz is not None:
-            return self
+            return self._eadata
         else:
             return self.values
 
@@ -629,6 +635,9 @@ def intersection(self, other):
               not other.freq.isAnchored() or
               (not self.is_monotonic or not other.is_monotonic)):
             result = Index.intersection(self, other)
+            # Invalidate the freq of `result`, which may not be correct at
+            # this point, depending on the values.
+            result.freq = None
             result = self._shallow_copy(result._values, name=result.name,
                                         tz=result.tz, freq=None)
             if result.freq is None:

pandas/core/internals/blocks.py

@@ -34,7 +34,8 @@
     _isna_compat, array_equivalent, is_null_datelike_scalar, isna, notna)
 
 import pandas.core.algorithms as algos
-from pandas.core.arrays import Categorical, ExtensionArray
+from pandas.core.arrays import (
+    Categorical, DatetimeArrayMixin as DatetimeArray, ExtensionArray)
 from pandas.core.base import PandasObject
 import pandas.core.common as com
 from pandas.core.indexes.datetimes import DatetimeIndex
@@ -2437,8 +2438,14 @@ def _try_coerce_args(self, values, other):
         """ provide coercion to our input arguments """
 
         if isinstance(other, ABCDatetimeIndex):
-            # to store DatetimeTZBlock as object
-            other = other.astype(object).values
+            # May get a DatetimeIndex here. Unbox it.
+            other = other.array
+
+        if isinstance(other, DatetimeArray):
+            # hit in pandas/tests/indexing/test_coercion.py
+            # ::TestWhereCoercion::test_where_series_datetime64[datetime64tz]
+            # when falling back to ObjectBlock.where
+            other = other.astype(object)
 
         return values, other
 
@@ -2985,7 +2992,8 @@ def _try_coerce_args(self, values, other):
         elif (is_null_datelike_scalar(other) or
               (lib.is_scalar(other) and isna(other))):
             other = tslibs.iNaT
-        elif isinstance(other, self._holder):
+        elif isinstance(other, (self._holder, DatetimeArray)):
+            # TODO: DatetimeArray check will be redundant after GH#24024
             if other.tz != self.values.tz:
                 raise ValueError("incompatible or non tz-aware value")
             other = _block_shape(other.asi8, ndim=self.ndim)

pandas/core/series.py

@@ -477,7 +477,10 @@ def _values(self):
         """
         Return the internal repr of this data.
         """
-        return self._data.internal_values()
+        result = self._data.internal_values()
+        if isinstance(result, DatetimeIndex):
+            result = result._eadata
+        return result
 
     def _formatting_values(self):
         """
@@ -1602,10 +1605,6 @@ def unique(self):
         Categories (3, object): [a < b < c]
         """
         result = super(Series, self).unique()
-        if isinstance(result, DatetimeIndex):
-            # TODO: This should be unnecessary after Series._values returns
-            #  DatetimeArray
-            result = result._eadata
         return result
 
     def drop_duplicates(self, keep='first', inplace=False):

pandas/tests/frame/common.py

@@ -1,9 +1,10 @@
 import numpy as np
 
-from pandas import compat
 from pandas.util._decorators import cache_readonly
-import pandas.util.testing as tm
+
 import pandas as pd
+from pandas import compat
+import pandas.util.testing as tm
 
 _seriesd = tm.getSeriesData()
 _tsd = tm.getTimeSeriesData()

pandas/tests/frame/conftest.py

@@ -1,10 +1,8 @@
-import pytest
-
 import numpy as np
+import pytest
 
-from pandas import compat
+from pandas import DataFrame, NaT, compat, date_range
 import pandas.util.testing as tm
-from pandas import DataFrame, date_range, NaT
 
 
 @pytest.fixture

pandas/tests/frame/test_alter_axes.py

@@ -2,22 +2,20 @@
 
 from __future__ import print_function
 
-import inspect
-import pytest
-
 from datetime import datetime, timedelta
+import inspect
 
 import numpy as np
+import pytest
+
+from pandas.compat import PY2, lrange
 
-from pandas.compat import lrange, PY2
-from pandas import (DataFrame, Series, Index, MultiIndex, RangeIndex,
-                    IntervalIndex, DatetimeIndex, Categorical, cut,
-                    Timestamp, date_range, to_datetime)
 from pandas.core.dtypes.common import (
-    is_object_dtype,
-    is_categorical_dtype,
-    is_interval_dtype)
+    is_categorical_dtype, is_interval_dtype, is_object_dtype)
 
+from pandas import (
+    Categorical, DataFrame, DatetimeIndex, Index, IntervalIndex, MultiIndex,
+    RangeIndex, Series, Timestamp, cut, date_range, to_datetime)
 import pandas.util.testing as tm
 
 

pandas/tests/frame/test_analytics.py

@@ -2,26 +2,26 @@
 
 from __future__ import print_function
 
-import warnings
 from datetime import timedelta
 import operator
-import pytest
-
 from string import ascii_lowercase
+import warnings
+
+import numpy as np
 from numpy import nan
 from numpy.random import randn
-import numpy as np
+import pytest
+
+from pandas.compat import PY35, lrange
+import pandas.util._test_decorators as td
 
-from pandas.compat import lrange, PY35
-from pandas import (compat, isna, notna, DataFrame, Series,
-                    MultiIndex, date_range, Timestamp, Categorical,
-                    to_datetime, to_timedelta)
 import pandas as pd
-import pandas.core.nanops as nanops
+from pandas import (
+    Categorical, DataFrame, MultiIndex, Series, Timestamp, compat, date_range,
+    isna, notna, to_datetime, to_timedelta)
 import pandas.core.algorithms as algorithms
-
+import pandas.core.nanops as nanops
 import pandas.util.testing as tm
-import pandas.util._test_decorators as td
 
 
 def assert_stat_op_calc(opname, alternative, frame, has_skipna=True,

pandas/tests/frame/test_api.py

@@ -2,27 +2,23 @@
 
 from __future__ import print_function
 
-import pytest
-
 # pylint: disable-msg=W0612,E1101
 from copy import deepcopy
 import pydoc
 
-from pandas.compat import range, lrange, long
-from pandas import compat
-
-from numpy.random import randn
 import numpy as np
+from numpy.random import randn
+import pytest
 
-from pandas import (DataFrame, Series, date_range, timedelta_range,
-                    Categorical, SparseDataFrame)
-import pandas as pd
-
-from pandas.util.testing import (assert_almost_equal,
-                                 assert_series_equal,
-                                 assert_frame_equal)
+from pandas.compat import long, lrange, range
 
+import pandas as pd
+from pandas import (
+    Categorical, DataFrame, Series, SparseDataFrame, compat, date_range,
+    timedelta_range)
 import pandas.util.testing as tm
+from pandas.util.testing import (
+    assert_almost_equal, assert_frame_equal, assert_series_equal)
 
 
 class SharedWithSparse(object):

pandas/tests/frame/test_apply.py

@@ -2,25 +2,24 @@
 
 from __future__ import print_function
 
-import pytest
-
-import operator
 from collections import OrderedDict
 from datetime import datetime
 from itertools import chain
-
+import operator
 import warnings
+
 import numpy as np
+import pytest
 
-from pandas import (notna, DataFrame, Series, MultiIndex, date_range,
-                    Timestamp, compat)
-import pandas as pd
 from pandas.core.dtypes.dtypes import CategoricalDtype
+
+import pandas as pd
+from pandas import (
+    DataFrame, MultiIndex, Series, Timestamp, compat, date_range, notna)
+from pandas.conftest import _get_cython_table_params
 from pandas.core.apply import frame_apply
-from pandas.util.testing import (assert_series_equal,
-                                 assert_frame_equal)
 import pandas.util.testing as tm
-from pandas.conftest import _get_cython_table_params
+from pandas.util.testing import assert_frame_equal, assert_series_equal
 
 
 @pytest.fixture

pandas/tests/frame/test_arithmetic.py

@@ -3,20 +3,19 @@
 from datetime import datetime
 import operator
 
-import pytest
 import numpy as np
+import pytest
 
 from pandas.compat import range
 
 import pandas as pd
-import pandas.util.testing as tm
-
 from pandas.tests.frame.common import _check_mixed_float, _check_mixed_int
-
+import pandas.util.testing as tm
 
 # -------------------------------------------------------------------
 # Comparisons
 
+
 class TestFrameComparisons(object):
     # Specifically _not_ flex-comparisons