msgpack: add optional msgpack.v5 support

The msgpack.v5 code located in msgpack_v5.go and
msgpack_v5_helper_test.go for tests. It is the same logic for
submodules.

An user can use msgpack.v5 with a build tag:
go_tarantool_msgpack_v5

The commit also unify typo conversions after decoding in tests. This
is necessary because  msgpack v2.9.2 decodes all numbers as uint[1]
or int[2], but msgpack.v5 decodes numbers as a MessagePack type[3].
There are additional differences when decoding types.

1. https://github.com/vmihailenco/msgpack/blob/23644a15054d8b9f8a9fca3e041f3419504b9c21/decode.go#L283
2. https://github.com/vmihailenco/msgpack/blob/23644a15054d8b9f8a9fca3e041f3419504b9c21/decode.go#L285
3. https://github.com/vmihailenco/msgpack/blob/233c977ae92b215a9aa7eb420bbe67da99f05ab6/decode.go#L410

Part of #124

Author: oleg-jukovec

CHANGELOG.md

@@ -18,6 +18,7 @@ Versioning](http://semver.org/spec/v2.0.0.html) except to the first release.
 - Prepared SQL statements (#117)
 - Context support for request objects (#48)
 - Streams and interactive transactions support (#101)
+- Optional msgpack.v5 usage (#124)
 
 ### Changed
 

README.md

@@ -26,6 +26,7 @@ faster than other packages according to public benchmarks.
 * [Documentation](#documentation)
   * [API reference](#api-reference)
   * [Walking\-through example](#walking-through-example)
+  * [msgpack.v5 migration](#msgpackv5-migration)
 * [Contributing](#contributing)
 * [Alternative connectors](#alternative-connectors)
 
@@ -68,7 +69,15 @@ This allows us to introduce new features without losing backward compatibility.
    go_tarantool_call_17
    ```
    **Note:** In future releases, `Call17` may be used as default `Call` behavior.
-3. To run fuzz tests with decimals, you can use the build tag:
+3. To replace usage of `msgpack.v2` with `msgpack.v5`, you can use the build
+   tag:
+   ```
+   go_tarantool_msgpack_v5
+   ```
+   **Note:** In future releases, `msgpack.v5` may be used by default. We recommend
+   to read [msgpack.v5 migration notes](#msgpackv5-migration) and try to
+   use msgpack.v5 before the changes.
+4. To run fuzz tests with decimals, you can use the build tag:
    ```
    go_tarantool_decimal_fuzzing
    ```
@@ -144,6 +153,31 @@ There are two parameters:
 * a space number (it could just as easily have been a space name), and
 * a tuple.
 
+### msgpack.v5 migration
+
+Most function names and argument types in `msgpack.v5` and `msgpack.v2`
+have not changed (in our code, we noticed changes in `EncodeInt`, `EncodeUint`
+and `RegisterExt`). But there are a lot of changes in a logic of encoding and
+decoding. On the plus side the migration seems easy, but on the minus side you
+need to be very careful.
+
+First of all, `EncodeInt8`, `EncodeInt16`, `EncodeInt32`, `EncodeInt64`
+and `EncodeUint*` analogues at `msgpack.v5` encode numbers as is without loss of
+type. In `msgpack.v2` the type of a number is reduced to a value.
+
+Secondly, a base decoding function does not convert numbers to `int64` or
+`uint64`. It converts numbers to an exact type defined by MessagePack. The
+change makes manual type conversions much more difficult and can lead to
+runtime errors with an old code. We do not recommend to use type conversions
+and give preference to `*Typed` functions (besides, it's faster).
+
+There are also changes in the logic that can lead to errors in the old code,
+[as example](https://github.com/vmihailenco/msgpack/issues/327). Although in
+`msgpack.v5` some functions for the logic tuning were added (see
+[UseLooseInterfaceDecoding](https://pkg.go.dev/github.com/vmihailenco/msgpack/v5#Decoder.UseLooseInterfaceDecoding), [UseCompactInts](https://pkg.go.dev/github.com/vmihailenco/msgpack/v5#Encoder.UseCompactInts) etc), it is still impossible
+to achieve full compliance of behavior between `msgpack.v5` and `msgpack.v2`. So
+we don't go this way. We use standart settings if it possible.
+
 ## Contributing
 
 See [the contributing guide](CONTRIBUTING.md) for detailed instructions on how

call_16_test.go

@@ -18,11 +18,11 @@ func TestConnection_Call(t *testing.T) {
 	defer conn.Close()
 
 	// Call16
-	resp, err = conn.Call("simple_incr", []interface{}{1})
+	resp, err = conn.Call("simple_concat", []interface{}{"1"})
 	if err != nil {
 		t.Errorf("Failed to use Call")
 	}
-	if resp.Data[0].([]interface{})[0].(uint64) != 2 {
+	if val, ok := resp.Data[0].([]interface{})[0].(string); !ok || val != "11" {
 		t.Errorf("result is not {{1}} : %v", resp.Data)
 	}
 }
@@ -34,18 +34,18 @@ func TestCallRequest(t *testing.T) {
 	conn := test_helpers.ConnectWithValidation(t, server, opts)
 	defer conn.Close()
 
-	req := NewCallRequest("simple_incr").Args([]interface{}{1})
+	req := NewCallRequest("simple_concat").Args([]interface{}{"1"})
 	resp, err = conn.Do(req).Get()
 	if err != nil {
 		t.Errorf("Failed to use Call")
 	}
-	if resp.Data[0].([]interface{})[0].(uint64) != 2 {
+	if val, ok := resp.Data[0].([]interface{})[0].(string); !ok || val != "11" {
 		t.Errorf("result is not {{1}} : %v", resp.Data)
 	}
 }
 
 func TestCallRequestCode(t *testing.T) {
-	req := NewCallRequest("simple_incrt")
+	req := NewCallRequest("simple_concat")
 	code := req.Code()
 	expected := Call16RequestCode
 	if code != int32(expected) {

call_17_test.go

@@ -18,11 +18,11 @@ func TestConnection_Call(t *testing.T) {
 	defer conn.Close()
 
 	// Call17
-	resp, err = conn.Call17("simple_incr", []interface{}{1})
+	resp, err = conn.Call17("simple_concat", []interface{}{"1"})
 	if err != nil {
 		t.Errorf("Failed to use Call")
 	}
-	if resp.Data[0].(uint64) != 2 {
+	if val, ok := resp.Data[0].(string); !ok || val != "11" {
 		t.Errorf("result is not {{1}} : %v", resp.Data)
 	}
 }
@@ -34,18 +34,18 @@ func TestCallRequest(t *testing.T) {
 	conn := test_helpers.ConnectWithValidation(t, server, opts)
 	defer conn.Close()
 
-	req := NewCallRequest("simple_incr").Args([]interface{}{1})
+	req := NewCallRequest("simple_concat").Args([]interface{}{"1"})
 	resp, err = conn.Do(req).Get()
 	if err != nil {
 		t.Errorf("Failed to use Call")
 	}
-	if resp.Data[0].(uint64) != 2 {
+	if val, ok := resp.Data[0].(string); !ok || val != "11" {
 		t.Errorf("result is not {{1}} : %v", resp.Data)
 	}
 }
 
 func TestCallRequestCode(t *testing.T) {
-	req := NewCallRequest("simple_incrt")
+	req := NewCallRequest("simple_concat")
 	code := req.Code()
 	expected := Call17RequestCode
 	if code != int32(expected) {

config.lua

@@ -80,7 +80,7 @@ box.once("init", function()
 
     --box.schema.user.grant('guest', 'read,write,execute', 'universe')
     box.schema.func.create('box.info')
-    box.schema.func.create('simple_incr')
+    box.schema.func.create('simple_concat')
 
     -- auth testing: access control
     box.schema.user.create('test', {password = 'test'})
@@ -106,10 +106,10 @@ local function func_name()
 end
 rawset(_G, 'func_name', func_name)
 
-local function simple_incr(a)
-    return a + 1
+local function simple_concat(a)
+    return a .. a
 end
-rawset(_G, 'simple_incr', simple_incr)
+rawset(_G, 'simple_concat', simple_concat)
 
 local function push_func(cnt)
     for i = 1, cnt do

datetime/datetime_test.go

@@ -64,10 +64,10 @@ func assertDatetimeIsEqual(t *testing.T, tuples []interface{}, tm time.Time) {
 		if len(tpl) != 2 {
 			t.Fatalf("Unexpected return value body (tuple len = %d)", len(tpl))
 		}
-		if val, ok := tpl[dtIndex].(Datetime); !ok || !val.ToTime().Equal(tm) {
+		if val, ok := toDatetime(tpl[dtIndex]); !ok || !val.ToTime().Equal(tm) {
 			t.Fatalf("Unexpected tuple %d field %v, expected %v",
 				dtIndex,
-				val.ToTime(),
+				val,
 				tm)
 		}
 	}
@@ -324,15 +324,18 @@ func (ev *Event) DecodeMsgpack(d *decoder) error {
 	if err != nil {
 		return err
 	}
-	ev.Datetime = res.(Datetime)
+	var ok bool
+	if ev.Datetime, ok = toDatetime(res); !ok {
+		return fmt.Errorf("datetime doesn't match")
+	}
 	return nil
 }
 
 func (c *Tuple2) EncodeMsgpack(e *encoder) error {
 	if err := e.EncodeArrayLen(3); err != nil {
 		return err
 	}
-	if err := encodeUint(e, uint64(c.Cid)); err != nil {
+	if err := e.EncodeUint64(uint64(c.Cid)); err != nil {
 		return err
 	}
 	if err := e.EncodeString(c.Orig); err != nil {
@@ -428,8 +431,8 @@ func TestCustomEncodeDecodeTuple1(t *testing.T) {
 	}
 
 	for i, tv := range []time.Time{tm1, tm2} {
-		dt := events[i].([]interface{})[1].(Datetime)
-		if !dt.ToTime().Equal(tv) {
+		dt, ok := toDatetime(events[i].([]interface{})[1])
+		if !ok || !dt.ToTime().Equal(tv) {
 			t.Fatalf("%v != %v", dt.ToTime(), tv)
 		}
 	}
@@ -501,7 +504,7 @@ func TestCustomEncodeDecodeTuple5(t *testing.T) {
 	if tpl, ok := resp.Data[0].([]interface{}); !ok {
 		t.Errorf("Unexpected body of Select")
 	} else {
-		if val, ok := tpl[0].(Datetime); !ok || !val.ToTime().Equal(tm) {
+		if val, ok := toDatetime(tpl[0]); !ok || !val.ToTime().Equal(tm) {
 			t.Fatalf("Unexpected body of Select")
 		}
 	}

datetime/msgpack.go

@@ -1,3 +1,6 @@
+//go:build !go_tarantool_msgpack_v5
+// +build !go_tarantool_msgpack_v5
+
 package datetime
 
 import (

datetime/msgpack_helper_test.go

@@ -1,14 +1,20 @@
+//go:build !go_tarantool_msgpack_v5
+// +build !go_tarantool_msgpack_v5
+
 package datetime_test
 
 import (
+	. "github.com/tarantool/go-tarantool/datetime"
+
 	"gopkg.in/vmihailenco/msgpack.v2"
 )
 
 type encoder = msgpack.Encoder
 type decoder = msgpack.Decoder
 
-func encodeUint(e *encoder, v uint64) error {
-	return e.EncodeUint(uint(v))
+func toDatetime(i interface{}) (dt Datetime, ok bool) {
+	dt, ok = i.(Datetime)
+	return
 }
 
 func marshal(v interface{}) ([]byte, error) {

datetime/msgpack_v5.go

@@ -0,0 +1,12 @@
+//go:build go_tarantool_msgpack_v5
+// +build go_tarantool_msgpack_v5
+
+package datetime
+
+import (
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+func init() {
+	msgpack.RegisterExt(datetime_extId, (*Datetime)(nil))
+}

datetime/msgpack_v5_helper_test.go

@@ -0,0 +1,28 @@
+//go:build go_tarantool_msgpack_v5
+// +build go_tarantool_msgpack_v5
+
+package datetime_test
+
+import (
+	. "github.com/tarantool/go-tarantool/datetime"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+type encoder = msgpack.Encoder
+type decoder = msgpack.Decoder
+
+func toDatetime(i interface{}) (dt Datetime, ok bool) {
+	var ptr *Datetime
+	if ptr, ok = i.(*Datetime); ok {
+		dt = *ptr
+	}
+	return
+}
+
+func marshal(v interface{}) ([]byte, error) {
+	return msgpack.Marshal(v)
+}
+
+func unmarshal(data []byte, v interface{}) error {
+	return msgpack.Unmarshal(data, v)
+}

decimal/decimal_test.go

@@ -60,8 +60,10 @@ func (t *TupleDecimal) DecodeMsgpack(d *decoder) error {
 	if err != nil {
 		return err
 	}
-	t.number = res.(Decimal)
-
+	var ok bool
+	if t.number, ok = toDecimal(res); !ok {
+		return fmt.Errorf("decimal doesn't match")
+	}
 	return nil
 }
 
@@ -347,7 +349,7 @@ func tupleValueIsDecimal(t *testing.T, tuples []interface{}, number decimal.Deci
 		if len(tpl) != 1 {
 			t.Fatalf("Unexpected return value body (tuple len)")
 		}
-		if val, ok := tpl[0].(Decimal); !ok || !val.Equal(number) {
+		if val, ok := toDecimal(tpl[0]); !ok || !val.Equal(number) {
 			t.Fatalf("Unexpected return value body (tuple 0 field)")
 		}
 	}
@@ -445,7 +447,10 @@ func TestMPDecode(t *testing.T) {
 			if err != nil {
 				t.Fatalf("Unmarshalling failed: %s", err.Error())
 			}
-			decActual := v.(Decimal)
+			decActual, ok := toDecimal(v)
+			if !ok {
+				t.Fatalf("Unable to convert to Decimal")
+			}
 
 			decExpected, err := decimal.NewFromString(testcase.numString)
 			if err != nil {

decimal/msgpack.go

@@ -1,3 +1,6 @@
+//go:build !go_tarantool_msgpack_v5
+// +build !go_tarantool_msgpack_v5
+
 package decimal
 
 import (

decimal/msgpack_helper_test.go

@@ -1,12 +1,22 @@
+//go:build !go_tarantool_msgpack_v5
+// +build !go_tarantool_msgpack_v5
+
 package decimal_test
 
 import (
+	. "github.com/tarantool/go-tarantool/decimal"
+
 	"gopkg.in/vmihailenco/msgpack.v2"
 )
 
 type encoder = msgpack.Encoder
 type decoder = msgpack.Decoder
 
+func toDecimal(i interface{}) (dec Decimal, ok bool) {
+	dec, ok = i.(Decimal)
+	return
+}
+
 func marshal(v interface{}) ([]byte, error) {
 	return msgpack.Marshal(v)
 }

decimal/msgpack_v5.go

@@ -0,0 +1,12 @@
+//go:build go_tarantool_msgpack_v5
+// +build go_tarantool_msgpack_v5
+
+package decimal
+
+import (
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+func init() {
+	msgpack.RegisterExt(decimalExtID, (*Decimal)(nil))
+}

decimal/msgpack_v5_helper_test.go

@@ -0,0 +1,28 @@
+//go:build go_tarantool_msgpack_v5
+// +build go_tarantool_msgpack_v5
+
+package decimal_test
+
+import (
+	. "github.com/tarantool/go-tarantool/decimal"
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+type encoder = msgpack.Encoder
+type decoder = msgpack.Decoder
+
+func toDecimal(i interface{}) (dec Decimal, ok bool) {
+	var ptr *Decimal
+	if ptr, ok = i.(*Decimal); ok {
+		dec = *ptr
+	}
+	return
+}
+
+func marshal(v interface{}) ([]byte, error) {
+	return msgpack.Marshal(v)
+}
+
+func unmarshal(data []byte, v interface{}) error {
+	return msgpack.Unmarshal(data, v)
+}