From bf827438da6d3d80b968938f61387c580042bcd7 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Fri, 9 Dec 2022 10:11:11 +0100 Subject: [PATCH 1/2] feat: add possibility to configure types of integers fields when initializing Point from `dict` structure --- influxdb_client/client/write/point.py | 49 +++++++++++++++++++++-- tests/test_point.py | 57 +++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 4 deletions(-) diff --git a/influxdb_client/client/write/point.py b/influxdb_client/client/write/point.py index 3bee403f..60ce7c40 100644 --- a/influxdb_client/client/write/point.py +++ b/influxdb_client/client/write/point.py @@ -99,6 +99,26 @@ def from_dict(dictionary: dict, write_precision: WritePrecision = DEFAULT_WRITE_ record_tag_keys=["location", "version"], record_field_keys=["pressure", "temperature"]) + Int Types: + The following example shows how to configure the types of integers fields. + It is useful when you want to serialize integers always as ``float`` to avoid ``field type conflict`` + or use ``unsigned 64-bit integer`` as the type for serialization. + + .. code-block:: python + + # Use custom dictionary structure + dict_structure = { + "measurement": "h2o_feet", + "tags": {"location": "coyote_creek"}, + "fields": { + "water_level": 1.0, + "some_counter": 108913123234 + }, + "time": 1 + } + + point = Point.from_dict(dict_structure, field_types={"some_counter": "uint"}) + :param dictionary: dictionary for serialize into data Point :param write_precision: sets the precision for the supplied time values :key record_measurement_key: key of dictionary with specified measurement @@ -106,8 +126,14 @@ def from_dict(dictionary: dict, write_precision: WritePrecision = DEFAULT_WRITE_ :key record_time_key: key of dictionary with specified timestamp :key record_tag_keys: list of dictionary keys to use as a tag :key record_field_keys: list of dictionary keys to use as a field + :key field_types: optional dictionary to specify types of serialized fields. Currently, is supported customization for integer types. + Possible integers types: + - ``int`` - serialize integers as "**Signed 64-bit integers**" - ``9223372036854775807i`` (default behaviour) + - ``uint`` - serialize integers as "**Unsigned 64-bit integers**" - ``9223372036854775807u`` + - ``float`` - serialize integers as "**IEEE-754 64-bit floating-point numbers**". Useful for unify number types in your pipeline to avoid field type conflict - ``9223372036854775807`` + The ``field_types`` can be also specified as part of incoming dictionary. For more info see an example above. :return: new data point - """ + """ # noqa: E501 measurement_ = kwargs.get('record_measurement_name', None) if measurement_ is None: measurement_ = dictionary[kwargs.get('record_measurement_key', 'measurement')] @@ -134,6 +160,19 @@ def from_dict(dictionary: dict, write_precision: WritePrecision = DEFAULT_WRITE_ record_time_key = kwargs.get('record_time_key', 'time') if record_time_key in dictionary: point.time(dictionary[record_time_key], write_precision=write_precision) + + _field_types = kwargs.get('field_types', {}) + if 'field_types' in dictionary: + _field_types = dictionary['field_types'] + # Map API fields types to Line Protocol types postfix: + # - int: 'i' + # - uint: 'u' + # - float: '' + point._field_types = dict(map( + lambda item: (item[0], 'i' if item[1] == 'int' else 'u' if item[1] == 'uint' else ''), + _field_types.items() + )) + return point def __init__(self, measurement_name): @@ -143,6 +182,7 @@ def __init__(self, measurement_name): self._name = measurement_name self._time = None self._write_precision = DEFAULT_WRITE_PRECISION + self._field_types = {} def time(self, time, write_precision=DEFAULT_WRITE_PRECISION): """ @@ -190,7 +230,7 @@ def to_line_protocol(self, precision=None): """ warnings.warn(message, SyntaxWarning) _tags = _append_tags(self._tags) - _fields = _append_fields(self._fields) + _fields = _append_fields(self._fields, self._field_types) if not _fields: return "" _time = _append_time(self._time, self._write_precision if precision is None else precision) @@ -227,7 +267,7 @@ def _append_tags(tags): return f"{',' if _return else ''}{','.join(_return)} " -def _append_fields(fields): +def _append_fields(fields, field_types): _return = [] for field, value in sorted(fields.items()): @@ -246,7 +286,8 @@ def _append_fields(fields): s = s[:-2] _return.append(f'{_escape_key(field)}={s}') elif (isinstance(value, int) or _np_is_subtype(value, 'int')) and not isinstance(value, bool): - _return.append(f'{_escape_key(field)}={str(value)}i') + _type = field_types.get(field, "i") + _return.append(f'{_escape_key(field)}={str(value)}{_type}') elif isinstance(value, bool): _return.append(f'{_escape_key(field)}={str(value).lower()}') elif isinstance(value, str): diff --git a/tests/test_point.py b/tests/test_point.py index 51552f02..cee602db 100644 --- a/tests/test_point.py +++ b/tests/test_point.py @@ -482,6 +482,63 @@ def test_from_dictionary_tolerant_to_missing_tags_and_fields(self): record_field_keys=["pressure", "temperature"]) self.assertEqual("sensor_pt859,location=warehouse_125 pressure=125i", point.to_line_protocol()) + def test_from_dictionary_uint(self): + dict_structure = { + "measurement": "h2o_feet", + "tags": {"location": "coyote_creek"}, + "fields": { + "water_level": 1.0, + "some_counter": 108913123234 + }, + "time": 1 + } + point = Point.from_dict(dict_structure, field_types={"some_counter": "uint"}) + self.assertEqual("h2o_feet,location=coyote_creek some_counter=108913123234u,water_level=1 1", + point.to_line_protocol()) + + def test_from_dictionary_int(self): + dict_structure = { + "measurement": "h2o_feet", + "tags": {"location": "coyote_creek"}, + "fields": { + "water_level": 1.0, + "some_counter": 108913123234 + }, + "time": 1 + } + point = Point.from_dict(dict_structure, field_types={"some_counter": "int"}) + self.assertEqual("h2o_feet,location=coyote_creek some_counter=108913123234i,water_level=1 1", + point.to_line_protocol()) + + def test_from_dictionary_float(self): + dict_structure = { + "measurement": "h2o_feet", + "tags": {"location": "coyote_creek"}, + "fields": { + "water_level": 1.0, + "some_counter": 108913123234 + }, + "time": 1 + } + point = Point.from_dict(dict_structure, field_types={"some_counter": "float"}) + self.assertEqual("h2o_feet,location=coyote_creek some_counter=108913123234,water_level=1 1", + point.to_line_protocol()) + + def test_from_dictionary_float_from_dict(self): + dict_structure = { + "measurement": "h2o_feet", + "tags": {"location": "coyote_creek"}, + "fields": { + "water_level": 1.0, + "some_counter": 108913123234 + }, + "field_types": {"some_counter": "float"}, + "time": 1 + } + point = Point.from_dict(dict_structure) + self.assertEqual("h2o_feet,location=coyote_creek some_counter=108913123234,water_level=1 1", + point.to_line_protocol()) + def test_static_measurement_name(self): dictionary = { "name": "sensor_pt859", From 8080bc8ea1ecaf4e6dc9e6b346fd7f5fe3ffe228 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Fri, 9 Dec 2022 10:15:10 +0100 Subject: [PATCH 2/2] docs: update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82c701bd..244bb58b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features 1. [#536](https://github.com/influxdata/influxdb-client-python/pull/536): Query to `CSV` skip empty lines +1. [#538](https://github.com/influxdata/influxdb-client-python/pull/538): Configure types of `integer` fields when initializing `Point` from `dict` structure ## 1.35.0 [2022-12-01]