From 050dd8a43d2aac03b7361bad35d1d43366f8516b Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 13 Jan 2021 12:26:48 -0500 Subject: [PATCH 1/5] Support JSON serialization of numpy datetime64 arrays --- packages/python/plotly/_plotly_utils/basevalidators.py | 2 +- packages/python/plotly/_plotly_utils/utils.py | 9 +++++++-- .../plotly/tests/test_optional/test_utils/test_utils.py | 9 +++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/python/plotly/_plotly_utils/basevalidators.py b/packages/python/plotly/_plotly_utils/basevalidators.py index 4685ed72802..e38d15647f6 100644 --- a/packages/python/plotly/_plotly_utils/basevalidators.py +++ b/packages/python/plotly/_plotly_utils/basevalidators.py @@ -146,7 +146,7 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False): # datatype. This works around cases like np.array([1, 2, '3']) where # numpy converts the integers to strings and returns array of dtype # ' Date: Wed, 13 Jan 2021 12:34:14 -0500 Subject: [PATCH 2/5] Support coercing pandas datetime DataFrames --- .../plotly/_plotly_utils/basevalidators.py | 7 +++ .../validators/test_pandas_series_input.py | 17 +++++- .../test_optional/test_utils/test_utils.py | 55 +++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/packages/python/plotly/_plotly_utils/basevalidators.py b/packages/python/plotly/_plotly_utils/basevalidators.py index e38d15647f6..b83f8524281 100644 --- a/packages/python/plotly/_plotly_utils/basevalidators.py +++ b/packages/python/plotly/_plotly_utils/basevalidators.py @@ -103,6 +103,13 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False): else: # DatetimeIndex v = v.to_pydatetime() + elif pd and isinstance(v, pd.DataFrame) and len(set(v.dtypes)) == 1: + dtype = v.dtypes[0] + if dtype.kind in numeric_kinds: + v = v.values + elif dtype.kind == "M": + v = [row.dt.to_pydatetime().tolist() for i, row in v.iterrows()] + if not isinstance(v, np.ndarray): # v has its own logic on how to convert itself into a numpy array if is_numpy_convertable(v): diff --git a/packages/python/plotly/_plotly_utils/tests/validators/test_pandas_series_input.py b/packages/python/plotly/_plotly_utils/tests/validators/test_pandas_series_input.py index 3783c4c1b79..26e4cbcf7cd 100644 --- a/packages/python/plotly/_plotly_utils/tests/validators/test_pandas_series_input.py +++ b/packages/python/plotly/_plotly_utils/tests/validators/test_pandas_series_input.py @@ -173,7 +173,7 @@ def test_color_validator_categorical(color_validator, color_categorical_pandas): np.testing.assert_array_equal(res, np.array(color_categorical_pandas)) -def test_data_array_validator_dates(data_array_validator, datetime_pandas, dates_array): +def test_data_array_validator_dates_series(data_array_validator, datetime_pandas, dates_array): res = data_array_validator.validate_coerce(datetime_pandas) @@ -185,3 +185,18 @@ def test_data_array_validator_dates(data_array_validator, datetime_pandas, dates # Check values np.testing.assert_array_equal(res, dates_array) + + +def test_data_array_validator_dates_dataframe(data_array_validator, datetime_pandas, dates_array): + + df = pd.DataFrame({"d": datetime_pandas}) + res = data_array_validator.validate_coerce(df) + + # Check type + assert isinstance(res, np.ndarray) + + # Check dtype + assert res.dtype == "object" + + # Check values + np.testing.assert_array_equal(res, dates_array.reshape(len(dates_array), 1)) diff --git a/packages/python/plotly/plotly/tests/test_optional/test_utils/test_utils.py b/packages/python/plotly/plotly/tests/test_optional/test_utils/test_utils.py index c902e193ea6..3cba9fd0565 100644 --- a/packages/python/plotly/plotly/tests/test_optional/test_utils/test_utils.py +++ b/packages/python/plotly/plotly/tests/test_optional/test_utils/test_utils.py @@ -257,6 +257,61 @@ def test_pandas_json_encoding(self): j6 = _json.dumps(ts.index, cls=utils.PlotlyJSONEncoder) assert j6 == '["2011-01-01T00:00:00", "2011-01-01T01:00:00"]' + def test_encode_customdata_datetime_series(self): + df = pd.DataFrame(dict(t=pd.to_datetime(["2010-01-01", "2010-01-02"]))) + + # 1D customdata + fig = Figure( + Scatter(x=df["t"], customdata=df["t"]), layout=dict(template="none") + ) + fig_json = _json.dumps( + fig, cls=utils.PlotlyJSONEncoder, separators=(",", ":"), sort_keys=True + ) + self.assertTrue( + fig_json.startswith( + '{"data":[{"customdata":["2010-01-01T00:00:00","2010-01-02T00:00:00"]' + ) + ) + + def test_encode_customdata_datetime_homogenous_dataframe(self): + df = pd.DataFrame(dict( + t1=pd.to_datetime(["2010-01-01", "2010-01-02"]), + t2=pd.to_datetime(["2011-01-01", "2011-01-02"]), + )) + # 2D customdata + fig = Figure( + Scatter(x=df["t1"], customdata=df[["t1", "t2"]]), layout=dict(template="none") + ) + fig_json = _json.dumps( + fig, cls=utils.PlotlyJSONEncoder, separators=(",", ":"), sort_keys=True + ) + self.assertTrue( + fig_json.startswith( + '{"data":[{"customdata":' + '[["2010-01-01T00:00:00","2011-01-01T00:00:00"],' + '["2010-01-02T00:00:00","2011-01-02T00:00:00"]' + ) + ) + + def test_encode_customdata_datetime_inhomogenous_dataframe(self): + df = pd.DataFrame(dict( + t=pd.to_datetime(["2010-01-01", "2010-01-02"]), + v=np.arange(2), + )) + # 2D customdata + fig = Figure( + Scatter(x=df["t"], customdata=df[["t", "v"]]), layout=dict(template="none") + ) + fig_json = _json.dumps( + fig, cls=utils.PlotlyJSONEncoder, separators=(",", ":"), sort_keys=True + ) + self.assertTrue( + fig_json.startswith( + '{"data":[{"customdata":' + '[["2010-01-01T00:00:00",0],["2010-01-02T00:00:00",1]]' + ) + ) + def test_numpy_masked_json_encoding(self): l = [1, 2, np.ma.core.masked] j1 = _json.dumps(l, cls=utils.PlotlyJSONEncoder) From 46d6a59d766032d37e15c582bd9f18e36da6c60a Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 13 Jan 2021 12:34:47 -0500 Subject: [PATCH 3/5] blacken --- .../validators/test_pandas_series_input.py | 8 +++++-- .../test_optional/test_utils/test_utils.py | 24 ++++++++++--------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/packages/python/plotly/_plotly_utils/tests/validators/test_pandas_series_input.py b/packages/python/plotly/_plotly_utils/tests/validators/test_pandas_series_input.py index 26e4cbcf7cd..53e05cd7d09 100644 --- a/packages/python/plotly/_plotly_utils/tests/validators/test_pandas_series_input.py +++ b/packages/python/plotly/_plotly_utils/tests/validators/test_pandas_series_input.py @@ -173,7 +173,9 @@ def test_color_validator_categorical(color_validator, color_categorical_pandas): np.testing.assert_array_equal(res, np.array(color_categorical_pandas)) -def test_data_array_validator_dates_series(data_array_validator, datetime_pandas, dates_array): +def test_data_array_validator_dates_series( + data_array_validator, datetime_pandas, dates_array +): res = data_array_validator.validate_coerce(datetime_pandas) @@ -187,7 +189,9 @@ def test_data_array_validator_dates_series(data_array_validator, datetime_pandas np.testing.assert_array_equal(res, dates_array) -def test_data_array_validator_dates_dataframe(data_array_validator, datetime_pandas, dates_array): +def test_data_array_validator_dates_dataframe( + data_array_validator, datetime_pandas, dates_array +): df = pd.DataFrame({"d": datetime_pandas}) res = data_array_validator.validate_coerce(df) diff --git a/packages/python/plotly/plotly/tests/test_optional/test_utils/test_utils.py b/packages/python/plotly/plotly/tests/test_optional/test_utils/test_utils.py index 3cba9fd0565..79ce6a7f320 100644 --- a/packages/python/plotly/plotly/tests/test_optional/test_utils/test_utils.py +++ b/packages/python/plotly/plotly/tests/test_optional/test_utils/test_utils.py @@ -274,13 +274,16 @@ def test_encode_customdata_datetime_series(self): ) def test_encode_customdata_datetime_homogenous_dataframe(self): - df = pd.DataFrame(dict( - t1=pd.to_datetime(["2010-01-01", "2010-01-02"]), - t2=pd.to_datetime(["2011-01-01", "2011-01-02"]), - )) + df = pd.DataFrame( + dict( + t1=pd.to_datetime(["2010-01-01", "2010-01-02"]), + t2=pd.to_datetime(["2011-01-01", "2011-01-02"]), + ) + ) # 2D customdata fig = Figure( - Scatter(x=df["t1"], customdata=df[["t1", "t2"]]), layout=dict(template="none") + Scatter(x=df["t1"], customdata=df[["t1", "t2"]]), + layout=dict(template="none"), ) fig_json = _json.dumps( fig, cls=utils.PlotlyJSONEncoder, separators=(",", ":"), sort_keys=True @@ -294,10 +297,9 @@ def test_encode_customdata_datetime_homogenous_dataframe(self): ) def test_encode_customdata_datetime_inhomogenous_dataframe(self): - df = pd.DataFrame(dict( - t=pd.to_datetime(["2010-01-01", "2010-01-02"]), - v=np.arange(2), - )) + df = pd.DataFrame( + dict(t=pd.to_datetime(["2010-01-01", "2010-01-02"]), v=np.arange(2),) + ) # 2D customdata fig = Figure( Scatter(x=df["t"], customdata=df[["t", "v"]]), layout=dict(template="none") @@ -337,8 +339,8 @@ def test_numpy_datetime64(self): j1 = _json.dumps(a, cls=utils.PlotlyJSONEncoder) assert ( j1 == '["2011-07-11T00:00:00.000000000", ' - '"2011-07-12T00:00:00.000000000", ' - '"2011-07-13T00:00:00.000000000"]' + '"2011-07-12T00:00:00.000000000", ' + '"2011-07-13T00:00:00.000000000"]' ) def test_pil_image_encoding(self): From fcba8a7035e78de810b8ddbbdade958aa54d0961 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 13 Jan 2021 12:42:46 -0500 Subject: [PATCH 4/5] Pre 1.0 pandas compatibilty --- packages/python/plotly/_plotly_utils/basevalidators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/python/plotly/_plotly_utils/basevalidators.py b/packages/python/plotly/_plotly_utils/basevalidators.py index b83f8524281..748f2ff70ed 100644 --- a/packages/python/plotly/_plotly_utils/basevalidators.py +++ b/packages/python/plotly/_plotly_utils/basevalidators.py @@ -104,7 +104,7 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False): # DatetimeIndex v = v.to_pydatetime() elif pd and isinstance(v, pd.DataFrame) and len(set(v.dtypes)) == 1: - dtype = v.dtypes[0] + dtype = v.dtypes.tolist()[0] if dtype.kind in numeric_kinds: v = v.values elif dtype.kind == "M": From 7d5023bbfa2fd5b0f08933828aad06c42f50bb10 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 13 Jan 2021 17:45:23 -0500 Subject: [PATCH 5/5] Only try `datetime_as_string` on datetime kinded numpy arrays --- packages/python/plotly/_plotly_utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/python/plotly/_plotly_utils/utils.py b/packages/python/plotly/_plotly_utils/utils.py index 4ce31257769..2ffd8e92978 100644 --- a/packages/python/plotly/_plotly_utils/utils.py +++ b/packages/python/plotly/_plotly_utils/utils.py @@ -184,7 +184,7 @@ def encode_as_numpy(obj): if obj is numpy.ma.core.masked: return float("nan") - elif isinstance(obj, numpy.ndarray): + elif isinstance(obj, numpy.ndarray) and obj.dtype.kind == "M": try: return numpy.datetime_as_string(obj).tolist() except TypeError: