From abbaa3b1f3f11bf88013fe6c534c64ec3167191e Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Tue, 24 Aug 2021 10:16:33 +0200 Subject: [PATCH 1/2] feat: add support for serialization FluxTable into JSON --- examples/README.md | 1 + examples/query_response_to_json.py | 26 +++ influxdb_client/client/flux_table.py | 27 ++- tests/query_output.json | 336 +++++++++++++++++++++++++++ tests/test_FluxCSVParser.py | 27 +++ tests/test_Thresholds.py | 2 +- 6 files changed, 417 insertions(+), 2 deletions(-) create mode 100644 examples/query_response_to_json.py create mode 100644 tests/query_output.json diff --git a/examples/README.md b/examples/README.md index 2cb4ac72..29efa6dd 100644 --- a/examples/README.md +++ b/examples/README.md @@ -11,6 +11,7 @@ ## Queries - [query.py](query.py) - How to query data into `FluxTable`s, `Stream` and `CSV` - [query_from_file.py](query_from_file.py) - How to use a Flux query defined in a separate file +- [query_response_to_json.py](query_response_to_json.py) - How to serialize Query response to JSON ## Management API diff --git a/examples/query_response_to_json.py b/examples/query_response_to_json.py new file mode 100644 index 00000000..e5228b74 --- /dev/null +++ b/examples/query_response_to_json.py @@ -0,0 +1,26 @@ +from influxdb_client import InfluxDBClient, Point +from influxdb_client.client.write_api import SYNCHRONOUS + +with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as client: + + """ + Prepare data + """ + _point1 = Point("my_measurement").tag("location", "Prague").field("temperature", 25.3) + _point2 = Point("my_measurement").tag("location", "New York").field("temperature", 24.3) + + client.write_api(write_options=SYNCHRONOUS).write(bucket="my-bucket", record=[_point1, _point2]) + + """ + Query: using Table structure + """ + tables = client.query_api().query('from(bucket:"my-bucket") |> range(start: -10m)') + + """ + Serialize to JSON + """ + import json + from influxdb_client.client.flux_table import FluxStructureEncoder + + output = json.dumps(tables, cls=FluxStructureEncoder, indent=2) + print(output) diff --git a/influxdb_client/client/flux_table.py b/influxdb_client/client/flux_table.py index abd80251..f35d6892 100644 --- a/influxdb_client/client/flux_table.py +++ b/influxdb_client/client/flux_table.py @@ -3,6 +3,7 @@ The data model consists of tables, records, columns. """ +from json import JSONEncoder class FluxStructure: @@ -11,8 +12,32 @@ class FluxStructure: pass +class FluxStructureEncoder(JSONEncoder): + """The FluxStructure encoder to encode query results to JSON.""" + + def default(self, obj): + """Return serializable objects for JSONEncoder.""" + import datetime + if isinstance(obj, FluxStructure): + return obj.__dict__ + elif isinstance(obj, (datetime.datetime, datetime.date)): + return obj.isoformat() + return super().default(obj) + + class FluxTable(FluxStructure): - """A table is set of records with a common set of columns and a group key.""" + """ + A table is set of records with a common set of columns and a group key. + + The table can be serialized into JSON by:: + + import json + from influxdb_client.client.flux_table import FluxStructureEncoder + + output = json.dumps(tables, cls=FluxStructureEncoder, indent=2) + print(output) + + """ def __init__(self) -> None: """Initialize defaults.""" diff --git a/tests/query_output.json b/tests/query_output.json new file mode 100644 index 00000000..4cbe36d0 --- /dev/null +++ b/tests/query_output.json @@ -0,0 +1,336 @@ +[ + { + "columns": [ + { + "default_value": "_result", + "group": false, + "data_type": "string", + "label": "result", + "index": 0 + }, + { + "default_value": "", + "group": false, + "data_type": "long", + "label": "table", + "index": 1 + }, + { + "default_value": "", + "group": true, + "data_type": "string", + "label": "_field", + "index": 2 + }, + { + "default_value": "", + "group": true, + "data_type": "string", + "label": "_measurement", + "index": 3 + }, + { + "default_value": "", + "group": true, + "data_type": "dateTime:RFC3339", + "label": "_start", + "index": 4 + }, + { + "default_value": "", + "group": true, + "data_type": "dateTime:RFC3339", + "label": "_stop", + "index": 5 + }, + { + "default_value": "", + "group": false, + "data_type": "dateTime:RFC3339", + "label": "_time", + "index": 6 + }, + { + "default_value": "", + "group": false, + "data_type": "double", + "label": "_value", + "index": 7 + }, + { + "default_value": "", + "group": true, + "data_type": "string", + "label": "tag", + "index": 8 + } + ], + "records": [ + { + "table": 0, + "values": { + "result": "_result", + "table": 0, + "_field": "value", + "_measurement": "python_client_test", + "_start": "2010-02-27T04:48:32.752600+00:00", + "_stop": "2020-02-27T16:48:32.752600+00:00", + "_time": "2020-02-27T16:20:00+00:00", + "_value": 2.0, + "tag": "test1" + } + }, + { + "table": 0, + "values": { + "result": "_result", + "table": 0, + "_field": "value", + "_measurement": "python_client_test", + "_start": "2010-02-27T04:48:32.752600+00:00", + "_stop": "2020-02-27T16:48:32.752600+00:00", + "_time": "2020-02-27T16:21:40+00:00", + "_value": 2.0, + "tag": "test1" + } + }, + { + "table": 0, + "values": { + "result": "_result", + "table": 0, + "_field": "value", + "_measurement": "python_client_test", + "_start": "2010-02-27T04:48:32.752600+00:00", + "_stop": "2020-02-27T16:48:32.752600+00:00", + "_time": "2020-02-27T16:23:20+00:00", + "_value": 2.0, + "tag": "test1" + } + }, + { + "table": 0, + "values": { + "result": "_result", + "table": 0, + "_field": "value", + "_measurement": "python_client_test", + "_start": "2010-02-27T04:48:32.752600+00:00", + "_stop": "2020-02-27T16:48:32.752600+00:00", + "_time": "2020-02-27T16:25:00+00:00", + "_value": 2.0, + "tag": "test1" + } + }, + { + "table": 0, + "values": { + "result": "_result", + "table": 0, + "_field": "value", + "_measurement": "python_client_test", + "_start": "2010-02-27T04:48:32.752600+00:00", + "_stop": "2020-02-27T16:48:32.752600+00:00", + "_time": "2020-02-27T16:26:40+00:00", + "_value": 2.0, + "tag": "test1" + } + }, + { + "table": 0, + "values": { + "result": "_result", + "table": 0, + "_field": "value", + "_measurement": "python_client_test", + "_start": "2010-02-27T04:48:32.752600+00:00", + "_stop": "2020-02-27T16:48:32.752600+00:00", + "_time": "2020-02-27T16:28:20+00:00", + "_value": 2.0, + "tag": "test1" + } + }, + { + "table": 0, + "values": { + "result": "_result", + "table": 0, + "_field": "value", + "_measurement": "python_client_test", + "_start": "2010-02-27T04:48:32.752600+00:00", + "_stop": "2020-02-27T16:48:32.752600+00:00", + "_time": "2020-02-27T16:30:00+00:00", + "_value": 2.0, + "tag": "test1" + } + } + ] + }, + { + "columns": [ + { + "default_value": "_result", + "group": false, + "data_type": "string", + "label": "result", + "index": 0 + }, + { + "default_value": "", + "group": false, + "data_type": "long", + "label": "table", + "index": 1 + }, + { + "default_value": "", + "group": true, + "data_type": "string", + "label": "_field", + "index": 2 + }, + { + "default_value": "", + "group": true, + "data_type": "string", + "label": "_measurement", + "index": 3 + }, + { + "default_value": "", + "group": true, + "data_type": "dateTime:RFC3339", + "label": "_start", + "index": 4 + }, + { + "default_value": "", + "group": true, + "data_type": "dateTime:RFC3339", + "label": "_stop", + "index": 5 + }, + { + "default_value": "", + "group": false, + "data_type": "dateTime:RFC3339", + "label": "_time", + "index": 6 + }, + { + "default_value": "", + "group": false, + "data_type": "double", + "label": "_value", + "index": 7 + }, + { + "default_value": "", + "group": true, + "data_type": "string", + "label": "tag", + "index": 8 + } + ], + "records": [ + { + "table": 1, + "values": { + "result": "_result", + "table": 1, + "_field": "value", + "_measurement": "python_client_test", + "_start": "2010-02-27T04:48:32.752600+00:00", + "_stop": "2020-02-27T16:48:32.752600+00:00", + "_time": "2020-02-27T16:20:00+00:00", + "_value": 2.0, + "tag": "test2" + } + }, + { + "table": 1, + "values": { + "result": "_result", + "table": 1, + "_field": "value", + "_measurement": "python_client_test", + "_start": "2010-02-27T04:48:32.752600+00:00", + "_stop": "2020-02-27T16:48:32.752600+00:00", + "_time": "2020-02-27T16:21:40+00:00", + "_value": 2.0, + "tag": "test2" + } + }, + { + "table": 1, + "values": { + "result": "_result", + "table": 1, + "_field": "value", + "_measurement": "python_client_test", + "_start": "2010-02-27T04:48:32.752600+00:00", + "_stop": "2020-02-27T16:48:32.752600+00:00", + "_time": "2020-02-27T16:23:20+00:00", + "_value": 2.0, + "tag": "test2" + } + }, + { + "table": 1, + "values": { + "result": "_result", + "table": 1, + "_field": "value", + "_measurement": "python_client_test", + "_start": "2010-02-27T04:48:32.752600+00:00", + "_stop": "2020-02-27T16:48:32.752600+00:00", + "_time": "2020-02-27T16:25:00+00:00", + "_value": 2.0, + "tag": "test2" + } + }, + { + "table": 1, + "values": { + "result": "_result", + "table": 1, + "_field": "value", + "_measurement": "python_client_test", + "_start": "2010-02-27T04:48:32.752600+00:00", + "_stop": "2020-02-27T16:48:32.752600+00:00", + "_time": "2020-02-27T16:26:40+00:00", + "_value": 2.0, + "tag": "test2" + } + }, + { + "table": 1, + "values": { + "result": "_result", + "table": 1, + "_field": "value", + "_measurement": "python_client_test", + "_start": "2010-02-27T04:48:32.752600+00:00", + "_stop": "2020-02-27T16:48:32.752600+00:00", + "_time": "2020-02-27T16:28:20+00:00", + "_value": 2.0, + "tag": "test2" + } + }, + { + "table": 1, + "values": { + "result": "_result", + "table": 1, + "_field": "value", + "_measurement": "python_client_test", + "_start": "2010-02-27T04:48:32.752600+00:00", + "_stop": "2020-02-27T16:48:32.752600+00:00", + "_time": "2020-02-27T16:30:00+00:00", + "_value": 2.0, + "tag": "test2" + } + } + ] + } +] \ No newline at end of file diff --git a/tests/test_FluxCSVParser.py b/tests/test_FluxCSVParser.py index 0f4c9bee..4cc1902f 100644 --- a/tests/test_FluxCSVParser.py +++ b/tests/test_FluxCSVParser.py @@ -5,6 +5,7 @@ from urllib3 import HTTPResponse from influxdb_client.client.flux_csv_parser import FluxCsvParser, FluxSerializationMode, FluxQueryException +from influxdb_client.client.flux_table import FluxStructureEncoder class FluxCsvParserTest(unittest.TestCase): @@ -185,6 +186,32 @@ def test_ParseInf(self): self.assertEqual(math.inf, tables[0].records[10]["le"]) self.assertEqual(-math.inf, tables[0].records[11]["le"]) + def test_to_json(self): + data = "#datatype,string,long,string,string,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,double,string\n" \ + "#group,false,false,true,true,true,true,false,false,true\n" \ + "#default,_result,,,,,,,,\n" \ + ",result,table,_field,_measurement,_start,_stop,_time,_value,tag\n" \ + ",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:20:00Z,2,test1\n" \ + ",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:21:40Z,2,test1\n" \ + ",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:23:20Z,2,test1\n" \ + ",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:25:00Z,2,test1\n" \ + ",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:26:40Z,2,test1\n" \ + ",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:28:20Z,2,test1\n" \ + ",,0,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:30:00Z,2,test1\n" \ + ",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:20:00Z,2,test2\n" \ + ",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:21:40Z,2,test2\n" \ + ",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:23:20Z,2,test2\n" \ + ",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:25:00Z,2,test2\n" \ + ",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:26:40Z,2,test2\n" \ + ",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:28:20Z,2,test2\n" \ + ",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:30:00Z,2,test2\n" + + tables = self._parse_to_tables(data=data) + with open('query_output.json', 'r') as file: + query_output = file.read() + import json + self.assertEqual(query_output, json.dumps(tables, cls=FluxStructureEncoder, indent=2)) + @staticmethod def _parse_to_tables(data: str): fp = BytesIO(str.encode(data)) diff --git a/tests/test_Thresholds.py b/tests/test_Thresholds.py index ef87a8b6..5dd28685 100644 --- a/tests/test_Thresholds.py +++ b/tests/test_Thresholds.py @@ -33,7 +33,7 @@ def test_threshold(self): }], } httpretty.register_uri(httpretty.GET, uri="http://localhost/api/v2/checks/01", status=200, - body=json.dumps(dictionary, indent=4), + body=json.dumps(dictionary, indent=2), adding_headers={'Content-Type': 'application/json'}) self.client = InfluxDBClient("http://localhost", "my-token", org="my-org", debug=True) checks_service = ChecksService(api_client=self.client.api_client) From 9d753a2cfdea9f690bb9d52d8df2ca0034b59a29 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Tue, 24 Aug 2021 10:23:27 +0200 Subject: [PATCH 2/2] docs: update CHANGELOG.md --- CHANGELOG.md | 1 + tests/test_FluxCSVParser.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 895d2643..64b0712c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features 1. [#319](https://github.com/influxdata/influxdb-client-python/pull/319): Add supports for array expressions in query parameters +2. [#320](https://github.com/influxdata/influxdb-client-python/pull/320): Add JSONEncoder to encode query results to JSON ### Bug Fixes 1. [#321](https://github.com/influxdata/influxdb-client-python/pull/321): Fixes return type for dashboard when `include=properties` is used diff --git a/tests/test_FluxCSVParser.py b/tests/test_FluxCSVParser.py index 4cc1902f..3112f0a9 100644 --- a/tests/test_FluxCSVParser.py +++ b/tests/test_FluxCSVParser.py @@ -207,7 +207,7 @@ def test_to_json(self): ",,1,value,python_client_test,2010-02-27T04:48:32.752600083Z,2020-02-27T16:48:32.752600083Z,2020-02-27T16:30:00Z,2,test2\n" tables = self._parse_to_tables(data=data) - with open('query_output.json', 'r') as file: + with open('tests/query_output.json', 'r') as file: query_output = file.read() import json self.assertEqual(query_output, json.dumps(tables, cls=FluxStructureEncoder, indent=2))