Support decimal data type in msgpack

This patch provides decimal support for all space operations and as
function return result. Decimal type was introduced in Tarantool 2.2.
See more about decimal type in [1] and [2].

To use decimal with github.com/shopspring/decimal in msgpack, import
tarantool/decimal submodule.

1. https://www.tarantool.io/en/doc/latest/book/box/data_model/
2. https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-decimal-type

Lua snippet for encoding number to MsgPack representation:

local decimal = require('decimal')

local function mp_encode_dec(num)
    local dec = msgpack.encode(decimal.new(num))
    return dec:gsub('.', function (c)
        return string.format('%02x', string.byte(c))
    end)
end
print(mp_encode_dec(-12.34)) -- 0xd6010201234d

Follows up tarantool/tarantool#692
Closes #96

Author: ligurio

CHANGELOG.md

@@ -16,6 +16,7 @@ Versioning](http://semver.org/spec/v2.0.0.html) except to the first release.
 - Support UUID type in msgpack (#90)
 - Go modules support (#91)
 - queue-utube handling (#85)
+- Support decimal type in msgpack (#96)
 
 ### Fixed
 

Makefile

@@ -15,6 +15,12 @@ test:
 	go clean -testcache
 	go test ./... -v -p 1	
 
+.PHONY: test-decimal
+test-decimal:
+	@echo "Running tests in decimal package"
+	go clean -testcache
+	go test ./decimal/ -v -p 1
+
 .PHONY: coverage
 coverage:
 	go clean -testcache

decimal/bcd.go

@@ -0,0 +1,163 @@
+// Package implements methods to encode and decode BCD.
+// BCD is a sequence of bytes representing decimal digits of the encoded number
+// (each byte has two decimal digits each encoded using 4-bit nibbles),
+// so byte >> 4 is the first digit and byte & 0x0f is the second digit. The
+// leftmost digit in the array is the most significant. The rightmost digit in
+// the array is the least significant.
+// The first byte of the BCD array contains the first digit of the number,
+// represented as follows:
+//
+//     |  4 bits           |  4 bits           |
+//        = 0x                = the 1st digit
+//
+// (The first nibble contains 0 if the decimal number has an even number of
+// digits.) The last byte of the BCD array contains the last digit of the
+// number and the final nibble, represented as follows:
+//
+//     |  4 bits           |  4 bits           |
+//        = the last digit    = nibble
+//
+// TODO: BitReader https://go.dev/play/p/Wyr_K9YAro
+package decimal
+
+import (
+	"errors"
+	_ "fmt"
+	"strconv"
+)
+
+var (
+	ErrNumberIsNotADecimal = errors.New("Number is not a decimal.")
+	ErrWrongExponentaRange = errors.New("Exponenta has a wrong range.")
+)
+
+const decimalMaxDigits = 38 // Maximum decimal digits taken by a decimal representation.
+const mpScaleIdx = 0
+
+var mpDecimalSign = map[rune]byte{
+	'+': 0x0a,
+	'-': 0x0b,
+}
+
+var mpIsDecimalNegative = map[byte]bool{
+	0x0a: false,
+	0x0b: true,
+	0x0c: false,
+	0x0d: true,
+	0x0e: false,
+	0x0f: false,
+}
+
+var hex_digit = map[rune]byte{
+	'1': 0x1,
+	'2': 0x2,
+	'3': 0x3,
+	'4': 0x4,
+	'5': 0x5,
+	'6': 0x6,
+	'7': 0x7,
+	'8': 0x8,
+	'9': 0x9,
+	'0': 0x0,
+}
+
+func MPEncodeNumberToBCD(buf string) []byte {
+	scale := 0
+	sign := '+'
+	// TODO: The first nibble contains 0 if the decimal number has an even number of digits.
+	nibbleIdx := 2 /* First nibble is for sign */
+	byteBuf := make([]byte, 1)
+	for i, ch := range buf {
+		// TODO: ignore leading zeroes
+		// Check for sign in a first nibble.
+		if (i == 0) && (ch == '-' || ch == '+') {
+			sign = ch
+			continue
+		}
+
+		// Remember a number of digits after the decimal point.
+		if ch == '.' {
+			scale = len(buf) - i - 1
+			continue
+		}
+
+		//digit := byte(ch) // TODO
+		digit := hex_digit[ch]
+		highNibble := nibbleIdx%2 != 0
+		lowByte := len(byteBuf) - 1
+		if highNibble {
+			digit = digit << 4
+			byteBuf = append(byteBuf, digit)
+		} else {
+			if nibbleIdx == 2 {
+				byteBuf[0] = digit
+			} else {
+				byteBuf[lowByte] = byteBuf[lowByte] | digit
+			}
+		}
+		//fmt.Printf("DEBUG: %x\n", byteBuf)
+		nibbleIdx += 1
+	}
+	if nibbleIdx%2 != 0 {
+		byteBuf = append(byteBuf, mpDecimalSign[sign])
+	} else {
+		lowByte := len(byteBuf) - 1
+		byteBuf[lowByte] = byteBuf[lowByte] | mpDecimalSign[sign]
+	}
+	byteBuf = append([]byte{byte(scale)}, byteBuf...)
+	//fmt.Printf("DEBUG: Encoded byteBuf %x\n", byteBuf)
+
+	return byteBuf
+}
+
+func highNibble(b byte) byte {
+	return b >> 4
+}
+
+func lowNibble(b byte) byte {
+	return b & 0x0f
+}
+
+func MPDecodeNumberFromBCD(bcdBuf []byte) ([]string, error) {
+	// TODO: move scale processing to outside of BCD package, it is not a
+	// part of BCD. Or rename package to PackedDecimal.
+	scale := int32(bcdBuf[mpScaleIdx])
+	//fmt.Printf("DEBUG: SCALE %d\n", scale)
+	// decimal scale is equal to -exponent and the exponent must be in
+	// range [ -decimalMaxDigits; decimalMaxDigits )
+	if scale < -decimalMaxDigits || scale >= decimalMaxDigits {
+		return nil, ErrWrongExponentaRange
+	}
+
+	// BCD buffer without a byte with scale.
+	bcdBuf = bcdBuf[mpScaleIdx+1:]
+	//fmt.Printf("DEBUG: BCDbuf without byte with scale %x\n", bcdBuf)
+	length := len(bcdBuf)
+	var digits []string
+	for i, bcd_byte := range bcdBuf {
+		// Skip leading zeros.
+		if len(digits) == 0 && int(bcd_byte) == 0 {
+			continue
+		}
+		if high := highNibble(bcd_byte); high != 0 {
+			digit := strconv.Itoa(int(high))
+			digits = append(digits, digit)
+		}
+		low := lowNibble(bcd_byte)
+		if int(i) != length-1 {
+			digit := strconv.Itoa(int(low))
+			digits = append(digits, digit)
+		}
+		/* TODO: Make sure every digit is less than 9 and bigger than 0 */
+	}
+
+	digits = append(digits[:scale+1], digits[scale:]...)
+	digits[scale] = "."
+	last_byte := bcdBuf[length-1]
+	sign := lowNibble(last_byte)
+	if mpIsDecimalNegative[sign] {
+		digits = append([]string{"-"}, digits...)
+	}
+
+	return digits, nil
+}

decimal/config.lua

@@ -0,0 +1,41 @@
+local decimal = require('decimal')
+local msgpack = require('msgpack')
+
+-- Do not set listen for now so connector won't be
+-- able to send requests until everything is configured.
+box.cfg{
+    work_dir = os.getenv("TEST_TNT_WORK_DIR"),
+}
+
+box.schema.user.create('test', { password = 'test' , if_not_exists = true })
+box.schema.user.grant('test', 'execute', 'universe', nil, { if_not_exists = true })
+
+local decimal_msgpack_supported = pcall(msgpack.encode, decimal.new(1))
+if not decimal_msgpack_supported then
+    error('Decimal unsupported, use Tarantool 2.2 or newer')
+end
+
+local s = box.schema.space.create('testDecimal', {
+    id = 524,
+    if_not_exists = true,
+})
+s:create_index('primary', {
+    type = 'TREE',
+    parts = {
+        {
+            field = 1,
+            type = 'decimal',
+        },
+    },
+    if_not_exists = true
+})
+s:truncate()
+
+box.schema.user.grant('test', 'read,write', 'space', 'testDecimal', { if_not_exists = true })
+
+-- Set listen only when every other thing is configured.
+box.cfg{
+    listen = os.getenv("TEST_TNT_LISTEN"),
+}
+
+require('console').start()

decimal/decimal.go

@@ -0,0 +1,67 @@
+// Package with support of Tarantool's decimal data type.
+//
+// Decimal data type supported in Tarantool since 2.2.
+//
+// Since: 1.6
+//
+// See also:
+//
+// * Tarantool MessagePack extensions https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-decimal-type
+//
+// * Tarantool data model https://www.tarantool.io/en/doc/latest/book/box/data_model/
+//
+// * Tarantool issue for support decimal type https://github.com/tarantool/tarantool/issues/692
+package decimal
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/shopspring/decimal"
+	"gopkg.in/vmihailenco/msgpack.v2"
+)
+
+// Decimal external type
+const decimal_extId = 1
+
+func encodeDecimal(e *msgpack.Encoder, v reflect.Value) error {
+	number := v.Interface().(decimal.Decimal)
+	dec := number.String()
+	bcdBuf := MPEncodeNumberToBCD(dec)
+	_, err := e.Writer().Write(bcdBuf)
+	if err != nil {
+		return fmt.Errorf("msgpack: can't write bytes to encoder writer: %w", err)
+	}
+
+	return nil
+}
+
+func decodeDecimal(d *msgpack.Decoder, v reflect.Value) error {
+	var bytesCount int = 4 // FIXME
+	b := make([]byte, bytesCount)
+
+	_, err := d.Buffered().Read(b)
+	if err != nil {
+		return fmt.Errorf("msgpack: can't read bytes on decimal decode: %w", err)
+	}
+
+	digits, err := MPDecodeNumberFromBCD(b)
+	if err != nil {
+		return err
+	}
+	str := strings.Join(digits, "")
+	dec, err := decimal.NewFromString(str)
+	if err != nil {
+		return err
+	}
+
+	v.Set(reflect.ValueOf(dec))
+
+	return nil
+}
+
+func init() {
+	msgpack.Register(reflect.TypeOf((*decimal.Decimal)(nil)).Elem(), encodeDecimal, decodeDecimal)
+	msgpack.RegisterExt(decimal_extId, (*decimal.Decimal)(nil))
+}