msgpack: support error extension type

Tarantool supports error extension type since version 2.4.1 [1],
encoding was introduced in Tarantool 2.10.0 [2]. This patch introduced
the support of Tarantool error extension type in msgpack decoders and
encoders.

Tarantool error extension type objects are decoded to
`tarantool.BoxError` type. `tarantool.BoxError` may be encoded to
Tarantool error extension type objects.

Error extension type internals are the same as errors extra information:
the only difference is that extra information is encoded as a separate
error dictionary field and error extension type objects is encoded as
MessagePack extension type objects.

Error extension type objects are parsed based on common
encoder/decoder rules. String fields are converted to either `str` or
`bytes` based on `encoding` mode.

The only way to receive an error extension type object from Tarantool is
to receive an explicitly built `box.error` object: either from
`return box.error.new(...)` or a tuple with it. All errors raised within
Tarantool (including those raised with `box.error(...)`) are encoded
based on the same rules as simple errors due to backward compatibility.

It is possible to create error extension type objects with Python code,
but it not likely to be really useful since most of their fields is
computed on error initialization on server side (even for custom error
types):

```
tarantool.BoxError(
    type='ClientError',
    file='[string "                     local err = box.error.ne..."]',
    line=1,
    message='Unknown error',
    errno=0,
    errcode=0,
)
```

1. tarantool/tarantool#4398
2. tarantool/tarantool#6433

Part of #232

Author: DifferentialOrange

CHANGELOG.md

@@ -137,6 +137,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Support iproto feature discovery (#206).
 - Backport ConnectionPool support for Python 3.6 (#232).
 - Support extra information for iproto errors (#232).
+- Error extension type support (#232).
 
 ### Changed
 - Bump msgpack requirement to 1.0.4 (PR #223).

doc/dev-guide.rst

@@ -83,6 +83,8 @@ they are represented with in-built and custom types:
     +-----------------------------+----+-------------+----+-----------------------------+
     | :obj:`uuid.UUID`            | -> | `UUID`_     | -> | :obj:`uuid.UUID`            |
     +-----------------------------+----+-------------+----+-----------------------------+
+    | :class:`tarantool.BoxError` | -> | `ERROR`_    | -> | :class:`tarantool.BoxError` |
+    +-----------------------------+----+-------------+----+-----------------------------+
     | :class:`tarantool.Datetime` | -> | `DATETIME`_ | -> | :class:`tarantool.Datetime` |
     +-----------------------------+----+-------------+----+-----------------------------+
     | :class:`tarantool.Interval` | -> | `INTERVAL`_ | -> | :class:`tarantool.Interval` |
@@ -109,5 +111,6 @@ and iterate through it as with any other serializable object.
 .. _extension types: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/
 .. _DECIMAL: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-decimal-type
 .. _UUID: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-uuid-type
+.. _ERROR: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-error-type
 .. _DATETIME: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-datetime-type
 .. _INTERVAL: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-interval-type

tarantool/__init__.py

@@ -42,6 +42,8 @@
 
 from tarantool.connection_pool import ConnectionPool, Mode
 
+from tarantool.types import BoxError
+
 __version__ = "0.9.0"
 
 
@@ -136,4 +138,4 @@ def connectmesh(addrs=({'host': 'localhost', 'port': 3301},), user=None,
 __all__ = ['connect', 'Connection', 'connectmesh', 'MeshConnection', 'Schema',
            'Error', 'DatabaseError', 'NetworkError', 'NetworkWarning',
            'SchemaError', 'dbapi', 'Datetime', 'Interval', 'IntervalAdjust',
-           'ConnectionPool', 'Mode']
+           'ConnectionPool', 'Mode', 'BoxError',]

tarantool/const.py

@@ -129,4 +129,4 @@
 # Tarantool 2.10 protocol version is 3
 CONNECTOR_IPROTO_VERSION = 3
 # List of connector-supported features
-CONNECTOR_FEATURES = []
+CONNECTOR_FEATURES = [IPROTO_FEATURE_ERROR_EXTENSION,]

tarantool/msgpack_ext/datetime.py

@@ -13,7 +13,7 @@
 `datetime`_ type id.
 """
 
-def encode(obj):
+def encode(obj, _):
     """
     Encode a datetime object.
 
@@ -28,7 +28,7 @@ def encode(obj):
 
     return obj.msgpack_encode()
 
-def decode(data):
+def decode(data, _):
     """
     Decode a datetime object.
 

tarantool/msgpack_ext/decimal.py

@@ -225,7 +225,7 @@ def strip_decimal_str(str_repr, scale, first_digit_ind):
     # Do not strips zeroes before the decimal point
     return str_repr
 
-def encode(obj):
+def encode(obj, _):
     """
     Encode a decimal object.
 
@@ -335,7 +335,7 @@ def add_str_digit(digit, digits_reverted, scale):
 
     digits_reverted.append(str(digit))
 
-def decode(data):
+def decode(data, _):
     """
     Decode a decimal object.
 

tarantool/msgpack_ext/error.py

@@ -0,0 +1,52 @@
+"""
+Tarantool `error`_ extension type support module.
+
+Refer to :mod:`~tarantool.msgpack_ext.types.error`.
+
+.. _error: https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-error-type
+"""
+
+from tarantool.types import (
+    encode_box_error,
+    decode_box_error,
+)
+
+EXT_ID = 3
+"""
+`error`_ type id.
+"""
+
+def encode(obj, packer):
+    """
+    Encode an error object.
+
+    :param obj: Error to encode.
+    :type obj: :class:`tarantool.BoxError`
+
+    :param packer: msgpack packer to encode error dictionary.
+    :type packer: :class:`msgpack.Packer`
+
+    :return: Encoded error.
+    :rtype: :obj:`bytes`
+    """
+
+    err_map = encode_box_error(obj)
+    return packer.pack(err_map)
+
+def decode(data, unpacker):
+    """
+    Decode an error object.
+
+    :param obj: Error to decode.
+    :type obj: :obj:`bytes`
+
+    :param unpacker: msgpack unpacker to decode error dictionary.
+    :type unpacker: :class:`msgpack.Unpacker`
+
+    :return: Decoded error.
+    :rtype: :class:`tarantool.BoxError`
+    """
+
+    unpacker.feed(data)
+    err_map = unpacker.unpack()
+    return decode_box_error(err_map)

tarantool/msgpack_ext/interval.py

@@ -13,7 +13,7 @@
 `datetime.interval`_ type id.
 """
 
-def encode(obj):
+def encode(obj, _):
     """
     Encode an interval object.
 
@@ -28,7 +28,7 @@ def encode(obj):
 
     return obj.msgpack_encode()
 
-def decode(data):
+def decode(data, _):
     """
     Decode an interval object.
 

tarantool/msgpack_ext/packer.py

@@ -8,28 +8,36 @@
 from uuid import UUID
 from msgpack import ExtType
 
+from tarantool.types import BoxError
 from tarantool.msgpack_ext.types.datetime import Datetime
 from tarantool.msgpack_ext.types.interval import Interval
 
 import tarantool.msgpack_ext.decimal as ext_decimal
 import tarantool.msgpack_ext.uuid as ext_uuid
+import tarantool.msgpack_ext.error as ext_error
 import tarantool.msgpack_ext.datetime as ext_datetime
 import tarantool.msgpack_ext.interval as ext_interval
 
 encoders = [
     {'type': Decimal,  'ext': ext_decimal },
     {'type': UUID,     'ext': ext_uuid    },
+    {'type': BoxError, 'ext': ext_error   },
     {'type': Datetime, 'ext': ext_datetime},
     {'type': Interval, 'ext': ext_interval},
 ]
 
-def default(obj):
+def default(obj, packer=None):
     """
     :class:`msgpack.Packer` encoder.
 
     :param obj: Object to encode.
     :type obj: :class:`decimal.Decimal` or :class:`uuid.UUID` or
-        :class:`tarantool.Datetime` or :class:`tarantool.Interval`
+         or :class:`tarantool.BoxError` or :class:`tarantool.Datetime`
+         or :class:`tarantool.Interval`
+
+    :param packer: msgpack packer to work with common types
+        (like dictionary in extended error payload)
+    :type packer: :class:`msgpack.Packer`, optional
 
     :return: Encoded value.
     :rtype: :class:`msgpack.ExtType`
@@ -39,5 +47,5 @@ def default(obj):
 
     for encoder in encoders:
         if isinstance(obj, encoder['type']):
-            return ExtType(encoder['ext'].EXT_ID, encoder['ext'].encode(obj))
+            return ExtType(encoder['ext'].EXT_ID, encoder['ext'].encode(obj, packer))
     raise TypeError("Unknown type: %r" % (obj,))

tarantool/msgpack_ext/unpacker.py

@@ -6,17 +6,19 @@
 
 import tarantool.msgpack_ext.decimal as ext_decimal
 import tarantool.msgpack_ext.uuid as ext_uuid
+import tarantool.msgpack_ext.error as ext_error
 import tarantool.msgpack_ext.datetime as ext_datetime
 import tarantool.msgpack_ext.interval as ext_interval
 
 decoders = {
     ext_decimal.EXT_ID : ext_decimal.decode ,
     ext_uuid.EXT_ID    : ext_uuid.decode    ,
+    ext_error.EXT_ID   : ext_error.decode   ,
     ext_datetime.EXT_ID: ext_datetime.decode,
     ext_interval.EXT_ID: ext_interval.decode,
 }
 
-def ext_hook(code, data):
+def ext_hook(code, data, unpacker=None):
     """
     :class:`msgpack.Unpacker` decoder.
 
@@ -26,13 +28,18 @@ def ext_hook(code, data):
     :param data: MessagePack extension type data.
     :type data: :obj:`bytes`
 
+    :param unpacker: msgpack unpacker to work with common types
+        (like dictionary in extended error payload)
+    :type unpacker: :class:`msgpack.Unpacker`, optional
+
     :return: Decoded value.
     :rtype: :class:`decimal.Decimal` or :class:`uuid.UUID` or
-        :class:`tarantool.Datetime` or :class:`tarantool.Interval`
+         or :class:`tarantool.BoxError` or :class:`tarantool.Datetime`
+         or :class:`tarantool.Interval`
 
     :raise: :exc:`NotImplementedError`
     """
 
     if code in decoders:
-        return decoders[code](data)
+        return decoders[code](data, unpacker)
     raise NotImplementedError("Unknown msgpack extension type code %d" % (code,))

tarantool/msgpack_ext/uuid.py

@@ -20,7 +20,7 @@
 `uuid`_ type id.
 """
 
-def encode(obj):
+def encode(obj, _):
     """
     Encode an UUID object.
 
@@ -33,7 +33,7 @@ def encode(obj):
 
     return obj.bytes
 
-def decode(data):
+def decode(data, _):
     """
     Decode an UUID object.
 

tarantool/request.py

@@ -63,6 +63,67 @@
 
 from tarantool.msgpack_ext.packer import default as packer_default
 
+def build_packer(conn):
+    """
+    Build packer to pack request.
+
+    :param conn: Request sender.
+    :type conn: :class:`~tarantool.Connection`
+
+    :rtype: :class:`msgpack.Packer`
+    """
+
+    packer_kwargs = dict()
+
+    # use_bin_type=True is default since msgpack-1.0.0.
+    #
+    # The option controls whether to pack binary (non-unicode)
+    # string values as mp_bin or as mp_str.
+    #
+    # The default behaviour of the Python 3 connector (since
+    # default encoding is "utf-8") is to pack bytes as mp_bin
+    # and Unicode strings as mp_str. encoding=None mode must
+    # be used to work with non-utf strings.
+    #
+    # encoding = 'utf-8'
+    #
+    # Python 3 -> Tarantool          -> Python 3
+    # str      -> mp_str (string)    -> str
+    # bytes    -> mp_bin (varbinary) -> bytes
+    #
+    # encoding = None
+    #
+    # Python 3 -> Tarantool          -> Python 3
+    # bytes    -> mp_str (string)    -> bytes
+    # str      -> mp_str (string)    -> bytes
+    #             mp_bin (varbinary) -> bytes
+    #
+    # msgpack-0.5.0 (and only this version) warns when the
+    # option is unset:
+    #
+    #  | FutureWarning: use_bin_type option is not specified.
+    #  | Default value of the option will be changed in future
+    #  | version.
+    #
+    # The option is supported since msgpack-0.4.0, so we can
+    # just always set it for all msgpack versions to get rid
+    # of the warning on msgpack-0.5.0 and to keep our
+    # behaviour on msgpack-1.0.0.
+    if conn.encoding is None:
+        packer_kwargs['use_bin_type'] = False
+    else:
+        packer_kwargs['use_bin_type'] = True
+
+    # We need configured packer to work with error extention
+    # type payload, but module do not provide access to self
+    # inside extension type packers.
+    packer_no_ext = msgpack.Packer(**packer_kwargs)
+    default = lambda obj: packer_default(obj, packer_no_ext)
+    packer_kwargs['default'] = default
+
+    return msgpack.Packer(**packer_kwargs)
+
+
 class Request(object):
     """
     Represents a single request to the server in compliance with the
@@ -87,50 +148,7 @@ def __init__(self, conn):
         self._body = ''
         self.response_class = Response
 
-        packer_kwargs = dict()
-
-        # use_bin_type=True is default since msgpack-1.0.0.
-        #
-        # The option controls whether to pack binary (non-unicode)
-        # string values as mp_bin or as mp_str.
-        #
-        # The default behaviour of the Python 3 connector (since
-        # default encoding is "utf-8") is to pack bytes as mp_bin
-        # and Unicode strings as mp_str. encoding=None mode must
-        # be used to work with non-utf strings.
-        #
-        # encoding = 'utf-8'
-        #
-        # Python 3 -> Tarantool          -> Python 3
-        # str      -> mp_str (string)    -> str
-        # bytes    -> mp_bin (varbinary) -> bytes
-        #
-        # encoding = None
-        #
-        # Python 3 -> Tarantool          -> Python 3
-        # bytes    -> mp_str (string)    -> bytes
-        # str      -> mp_str (string)    -> bytes
-        #             mp_bin (varbinary) -> bytes
-        #
-        # msgpack-0.5.0 (and only this version) warns when the
-        # option is unset:
-        #
-        #  | FutureWarning: use_bin_type option is not specified.
-        #  | Default value of the option will be changed in future
-        #  | version.
-        #
-        # The option is supported since msgpack-0.4.0, so we can
-        # just always set it for all msgpack versions to get rid
-        # of the warning on msgpack-0.5.0 and to keep our
-        # behaviour on msgpack-1.0.0.
-        if conn.encoding is None:
-            packer_kwargs['use_bin_type'] = False
-        else:
-            packer_kwargs['use_bin_type'] = True
-
-        packer_kwargs['default'] = packer_default
-
-        self.packer = msgpack.Packer(**packer_kwargs)
+        self.packer = build_packer(conn)
 
     def _dumps(self, src):
         """