diff --git a/doc/requirements.txt b/doc/requirements.txt index 6e2f295857a..fb71acdb790 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -31,3 +31,4 @@ umap-learn==0.5.1 pooch wget nbconvert==5.6.1 +orjson diff --git a/packages/python/plotly/_plotly_utils/basevalidators.py b/packages/python/plotly/_plotly_utils/basevalidators.py index 748f2ff70ed..1dc3ea48677 100644 --- a/packages/python/plotly/_plotly_utils/basevalidators.py +++ b/packages/python/plotly/_plotly_utils/basevalidators.py @@ -53,7 +53,7 @@ def to_scalar_or_list(v): return v -def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False): +def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False): """ Convert an array-like value into a read-only numpy array @@ -89,7 +89,13 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False): # u: unsigned int, i: signed int, f: float numeric_kinds = {"u", "i", "f"} - kind_default_dtypes = {"u": "uint32", "i": "int32", "f": "float64", "O": "object"} + kind_default_dtypes = { + "u": "uint32", + "i": "int32", + "f": "float64", + "O": "object", + "U": "U", + } # Handle pandas Series and Index objects if pd and isinstance(v, (pd.Series, pd.Index)): @@ -113,18 +119,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False): if not isinstance(v, np.ndarray): # v has its own logic on how to convert itself into a numpy array if is_numpy_convertable(v): - return copy_to_readonly_numpy_array( + return copy_to_readonly_numpy_array_or_list( np.array(v), kind=kind, force_numeric=force_numeric ) else: # v is not homogenous array - v_list = [to_scalar_or_list(e) for e in v] - - # Lookup dtype for requested kind, if any - dtype = kind_default_dtypes.get(first_kind, None) - - # construct new array from list - new_v = np.array(v_list, order="C", dtype=dtype) + return [to_scalar_or_list(e) for e in v] elif v.dtype.kind in numeric_kinds: # v is a homogenous numeric array if kind and v.dtype.kind not in kind: @@ -135,6 +135,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False): else: # Either no kind was requested or requested kind is satisfied new_v = np.ascontiguousarray(v.copy()) + elif v.dtype.kind == "O": + if kind: + dtype = kind_default_dtypes.get(first_kind, None) + return np.array(v, dtype=dtype) + else: + return v.tolist() else: # v is a non-numeric homogenous array new_v = v.copy() @@ -149,12 +155,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False): if "U" not in kind: # Force non-numeric arrays to have object type # -------------------------------------------- - # Here we make sure that non-numeric arrays have the object - # datatype. This works around cases like np.array([1, 2, '3']) where + # Here we make sure that non-numeric arrays become lists + # This works around cases like np.array([1, 2, '3']) where # numpy converts the integers to strings and returns array of dtype # ' " ) diff --git a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py new file mode 100644 index 00000000000..70d6803f1ac --- /dev/null +++ b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py @@ -0,0 +1,204 @@ +import pytest +import plotly.io.json as pio +import plotly.graph_objects as go +import numpy as np +import pandas as pd +import json +import datetime +import sys +from pytz import timezone +from _plotly_utils.optional_imports import get_module + +orjson = get_module("orjson") + +eastern = timezone("US/Eastern") + + +# Testing helper +def build_json_opts(pretty=False): + opts = {"sort_keys": True} + if pretty: + opts["indent"] = 2 + else: + opts["separators"] = (",", ":") + return opts + + +def to_json_test(value, pretty=False): + return json.dumps(value, **build_json_opts(pretty=pretty)) + + +def isoformat_test(dt_value): + if isinstance(dt_value, np.datetime64): + return str(dt_value) + elif isinstance(dt_value, datetime.datetime): + return dt_value.isoformat() + else: + raise ValueError("Unsupported date type: {}".format(type(dt_value))) + + +def build_test_dict(value): + return dict(a=value, b=[3, value], c=dict(Z=value)) + + +def build_test_dict_string(value_string, pretty=False): + if pretty: + non_pretty_str = build_test_dict_string(value_string, pretty=False) + return to_json_test(json.loads(non_pretty_str), pretty=True) + else: + value_string = str(value_string).replace(" ", "") + return """{"a":%s,"b":[3,%s],"c":{"Z":%s}}""" % tuple([value_string] * 3) + + +def check_roundtrip(value, engine, pretty): + encoded = pio.to_json_plotly(value, engine=engine, pretty=pretty) + decoded = pio.from_json_plotly(encoded, engine=engine) + reencoded = pio.to_json_plotly(decoded, engine=engine, pretty=pretty) + assert encoded == reencoded + + # Check from_plotly_json with bytes on Python 3 + if sys.version_info.major == 3: + encoded_bytes = encoded.encode("utf8") + decoded_from_bytes = pio.from_json_plotly(encoded_bytes, engine=engine) + assert decoded == decoded_from_bytes + + +# Fixtures +if orjson is not None: + engines = ["json", "orjson", "auto"] +else: + engines = ["json", "auto"] + + +@pytest.fixture(scope="module", params=engines) +def engine(request): + return request.param + + +@pytest.fixture(scope="module", params=[False]) +def pretty(request): + return request.param + + +@pytest.fixture(scope="module", params=["float64", "int32", "uint32"]) +def graph_object(request): + return request.param + + +@pytest.fixture(scope="module", params=["float64", "int32", "uint32"]) +def numeric_numpy_array(request): + dtype = request.param + return np.linspace(-5, 5, 4, dtype=dtype) + + +@pytest.fixture(scope="module") +def object_numpy_array(request): + return np.array(["a", 1, [2, 3]]) + + +@pytest.fixture(scope="module") +def numpy_unicode_array(request): + return np.array(["A", "BB", "CCC"], dtype="U") + + +@pytest.fixture( + scope="module", + params=[ + datetime.datetime(2003, 7, 12, 8, 34, 22), + datetime.datetime.now(), + np.datetime64(datetime.datetime.utcnow()), + pd.Timestamp(datetime.datetime.now()), + eastern.localize(datetime.datetime(2003, 7, 12, 8, 34, 22)), + eastern.localize(datetime.datetime.now()), + pd.Timestamp(datetime.datetime.now(), tzinfo=eastern), + ], +) +def datetime_value(request): + return request.param + + +@pytest.fixture( + params=[ + list, # plain list of datetime values + lambda a: pd.DatetimeIndex(a), # Pandas DatetimeIndex + lambda a: pd.Series(pd.DatetimeIndex(a)), # Pandas Datetime Series + lambda a: pd.DatetimeIndex(a).values, # Numpy datetime64 array + lambda a: np.array(a, dtype="object"), # Numpy object array of datetime + ] +) +def datetime_array(request, datetime_value): + return request.param([datetime_value] * 3) + + +# Encoding tests +def test_graph_object_input(engine, pretty): + scatter = go.Scatter(x=[1, 2, 3], y=np.array([4, 5, 6])) + result = pio.to_json_plotly(scatter, engine=engine) + expected = """{"type":"scatter","x":[1,2,3],"y":[4,5,6]}""" + assert result == expected + check_roundtrip(result, engine=engine, pretty=pretty) + + +def test_numeric_numpy_encoding(numeric_numpy_array, engine, pretty): + value = build_test_dict(numeric_numpy_array) + result = pio.to_json_plotly(value, engine=engine, pretty=pretty) + + array_str = to_json_test(numeric_numpy_array.tolist()) + expected = build_test_dict_string(array_str, pretty=pretty) + assert result == expected + check_roundtrip(result, engine=engine, pretty=pretty) + + +def test_numpy_unicode_encoding(numpy_unicode_array, engine, pretty): + value = build_test_dict(numpy_unicode_array) + result = pio.to_json_plotly(value, engine=engine, pretty=pretty) + + array_str = to_json_test(numpy_unicode_array.tolist()) + expected = build_test_dict_string(array_str) + assert result == expected + check_roundtrip(result, engine=engine, pretty=pretty) + + +def test_object_numpy_encoding(object_numpy_array, engine, pretty): + value = build_test_dict(object_numpy_array) + result = pio.to_json_plotly(value, engine=engine, pretty=pretty) + + array_str = to_json_test(object_numpy_array.tolist()) + expected = build_test_dict_string(array_str) + assert result == expected + check_roundtrip(result, engine=engine, pretty=pretty) + + +def test_datetime(datetime_value, engine, pretty): + value = build_test_dict(datetime_value) + result = pio.to_json_plotly(value, engine=engine, pretty=pretty) + expected = build_test_dict_string('"{}"'.format(isoformat_test(datetime_value))) + assert result == expected + check_roundtrip(result, engine=engine, pretty=pretty) + + +def test_datetime_arrays(datetime_array, engine, pretty): + value = build_test_dict(datetime_array) + result = pio.to_json_plotly(value, engine=engine) + + def to_str(v): + try: + v = v.isoformat(sep="T") + except (TypeError, AttributeError): + pass + + return str(v) + + if isinstance(datetime_array, list): + dt_values = [to_str(d) for d in datetime_array] + elif isinstance(datetime_array, pd.Series): + dt_values = [to_str(d) for d in datetime_array.dt.to_pydatetime().tolist()] + elif isinstance(datetime_array, pd.DatetimeIndex): + dt_values = [to_str(d) for d in datetime_array.to_pydatetime().tolist()] + else: # numpy datetime64 array + dt_values = [to_str(d) for d in datetime_array] + + array_str = to_json_test(dt_values) + expected = build_test_dict_string(array_str) + assert result == expected + check_roundtrip(result, engine=engine, pretty=pretty) diff --git a/packages/python/plotly/plotly/tests/test_optional/test_offline/test_offline.py b/packages/python/plotly/plotly/tests/test_optional/test_offline/test_offline.py index a50fa4e7675..dd9204947f5 100644 --- a/packages/python/plotly/plotly/tests/test_optional/test_offline/test_offline.py +++ b/packages/python/plotly/plotly/tests/test_optional/test_offline/test_offline.py @@ -10,6 +10,7 @@ import pytest import plotly +import plotly.io as pio from plotly import optional_imports matplotlylib = optional_imports.get_module("plotly.matplotlylib") @@ -77,12 +78,8 @@ def test_default_mpl_plot_generates_expected_html(self): data = figure["data"] layout = figure["layout"] - data_json = _json.dumps( - data, cls=plotly.utils.PlotlyJSONEncoder, sort_keys=True - ) - layout_json = _json.dumps( - layout, cls=plotly.utils.PlotlyJSONEncoder, sort_keys=True - ) + data_json = pio.json.to_json_plotly(data) + layout_json = pio.json.to_json_plotly(layout) html = self._read_html(plotly.offline.plot_mpl(fig)) # blank out uid before comparisons diff --git a/packages/python/plotly/plotly/tests/test_optional/test_px/test_px.py b/packages/python/plotly/plotly/tests/test_optional/test_px/test_px.py index c3236de32ce..03f10722794 100644 --- a/packages/python/plotly/plotly/tests/test_optional/test_px/test_px.py +++ b/packages/python/plotly/plotly/tests/test_optional/test_px/test_px.py @@ -37,8 +37,8 @@ def test_custom_data_scatter(): hover_data=["petal_length", "petal_width"], custom_data=["species_id", "species"], ) - assert np.all(fig.data[0].customdata[:, 0] == iris.species_id) - assert fig.data[0].customdata.shape[1] == 4 + assert [e[0] for e in fig.data[0].customdata] == iris.species_id.to_list() + assert len(fig.data[0].customdata[0]) == 4 # Hover and custom data, with repeated arguments fig = px.scatter( iris, @@ -47,8 +47,8 @@ def test_custom_data_scatter(): hover_data=["petal_length", "petal_width", "species_id"], custom_data=["species_id", "species"], ) - assert np.all(fig.data[0].customdata[:, 0] == iris.species_id) - assert fig.data[0].customdata.shape[1] == 4 + assert [e[0] for e in fig.data[0].customdata] == iris.species_id.tolist() + assert len(fig.data[0].customdata[0]) == 4 assert ( fig.data[0].hovertemplate == "sepal_width=%{x}
sepal_length=%{y}
petal_length=%{customdata[2]}
petal_width=%{customdata[3]}
species_id=%{customdata[0]}" diff --git a/packages/python/plotly/plotly/tests/test_optional/test_px/test_px_functions.py b/packages/python/plotly/plotly/tests/test_optional/test_px/test_px_functions.py index a75a45f43d5..fad79b5ded4 100644 --- a/packages/python/plotly/plotly/tests/test_optional/test_px/test_px_functions.py +++ b/packages/python/plotly/plotly/tests/test_optional/test_px/test_px_functions.py @@ -229,9 +229,9 @@ def test_sunburst_treemap_with_path_color(): df["hover"] = [el.lower() for el in vendors] fig = px.sunburst(df, path=path, color="calls", hover_data=["hover"]) custom = fig.data[0].customdata - assert np.all(custom[:8, 0] == df["hover"]) - assert np.all(custom[8:, 0] == "(?)") - assert np.all(custom[:8, 1] == df["calls"]) + assert [el[0] for el in custom[:8]] == df["hover"].tolist() + assert [el[0] for el in custom[8:]] == ["(?)"] * 7 + assert [el[1] for el in custom[:8]] == df["calls"].tolist() # Discrete color fig = px.sunburst(df, path=path, color="vendors") diff --git a/packages/python/plotly/plotly/tests/test_optional/test_px/test_px_input.py b/packages/python/plotly/plotly/tests/test_optional/test_px/test_px_input.py index 477e7dbcb04..0dce0ae663e 100644 --- a/packages/python/plotly/plotly/tests/test_optional/test_px/test_px_input.py +++ b/packages/python/plotly/plotly/tests/test_optional/test_px/test_px_input.py @@ -126,7 +126,7 @@ def test_repeated_name(): hover_data=["petal_length", "petal_width", "species_id"], custom_data=["species_id", "species"], ) - assert fig.data[0].customdata.shape[1] == 4 + assert len(fig.data[0].customdata[0]) == 4 def test_arrayattrable_numpy(): diff --git a/packages/python/plotly/templategen/__init__.py b/packages/python/plotly/templategen/__init__.py index 9dcc21a765b..bdcf8e10448 100644 --- a/packages/python/plotly/templategen/__init__.py +++ b/packages/python/plotly/templategen/__init__.py @@ -1,5 +1,4 @@ -from plotly.utils import PlotlyJSONEncoder -import json +from plotly.io.json import to_json_plotly import os from templategen.definitions import builders @@ -20,4 +19,4 @@ ), "w", ) as f: - plotly_schema = json.dump(template, f, cls=PlotlyJSONEncoder) + f.write(to_json_plotly(template))